Code Review / csit.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | review | tree
raw | inline | side by side
Trending: Add 3n-alt - fixes
[csit.git] / resources / tools / dash / app / pal / trending / layout.py
diff --git a/resources/tools/dash/app/pal/trending/layout.py b/resources/tools/dash/app/pal/trending/layout.py
index 9153081..afa459b 100644 (file)
--- a/resources/tools/dash/app/pal/trending/layout.py
+++ b/resources/tools/dash/app/pal/trending/layout.py
@@ -14,27 +14,35 @@
 """Plotly Dash HTML layout override.
 """
 
 """Plotly Dash HTML layout override.
 """
 
-
-import plotly.graph_objects as go
+import pandas as pd
 
 from dash import dcc
 from dash import html
 from dash import callback_context, no_update
 
 from dash import dcc
 from dash import html
 from dash import callback_context, no_update
-from dash import Input, Output, State, callback
+from dash import Input, Output, State
 from dash.exceptions import PreventUpdate
 from yaml import load, FullLoader, YAMLError
 from datetime import datetime, timedelta
 
 from dash.exceptions import PreventUpdate
 from yaml import load, FullLoader, YAMLError
 from datetime import datetime, timedelta
 
-from pprint import pformat
-
-from .data import read_data
+from ..data.data import Data
+from .graphs import graph_trending, graph_hdrh_latency, \
+    select_trending_data
 
 
 class Layout:
     """
     """
 
 
 
 class Layout:
     """
     """
 
-    def __init__(self, app, html_layout_file, spec_file, graph_layout_file):
+    STYLE_HIDEN = {"display": "none"}
+    STYLE_BLOCK = {"display": "block", "vertical-align": "top"}
+    STYLE_INLINE ={
+        "display": "inline-block",
+        "vertical-align": "top"
+    }
+    NO_GRAPH = {"data": [], "layout": {}, "frames": []}
+
+    def __init__(self, app, html_layout_file, spec_file, graph_layout_file,
+        data_spec_file):
         """
         """
 
         """
         """
 
@@ -43,9 +51,20 @@ class Layout:
         self._html_layout_file = html_layout_file
         self._spec_file = spec_file
         self._graph_layout_file = graph_layout_file
         self._html_layout_file = html_layout_file
         self._spec_file = spec_file
         self._graph_layout_file = graph_layout_file
+        self._data_spec_file = data_spec_file
 
         # Read the data:
 
         # Read the data:
-        self._data = read_data()
+        data_mrr = Data(
+            data_spec_file=self._data_spec_file,
+            debug=True
+        ).read_trending_mrr()
+
+        data_ndrpdr = Data(
+            data_spec_file=self._data_spec_file,
+            debug=True
+        ).read_trending_ndrpdr()
+
+        self._data = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
 
         # Read from files:
         self._html_layout = ""
 
         # Read from files:
         self._html_layout = ""
@@ -105,6 +124,10 @@ class Layout:
     def data(self):
         return self._data
 
     def data(self):
         return self._data
 
+    @property
+    def layout(self):
+        return self._graph_layout
+
     def add_content(self):
         """
         """
     def add_content(self):
         """
         """
@@ -139,8 +162,7 @@ class Layout:
             ],
             style={
                 "display": "inline-block",
             ],
             style={
                 "display": "inline-block",
-                "width": "18%",
-                "padding": "5px"
+                "width": "20%"
             }
         )
 
             }
         )
 
@@ -150,21 +172,107 @@ class Layout:
         return html.Div(
             id="div-plotting-area",
             children=[
         return html.Div(
             id="div-plotting-area",
             children=[
-                dcc.Loading(
-                    id="loading-graph",
-                    children=[
-                        dcc.Graph(
-                            id="graph"
-                        )
-                    ],
-                    type="circle"
-                )
+                html.Table(children=[
+                    html.Tr(
+                        id="div-tput",
+                        style=self.STYLE_HIDEN,
+                        children=[
+                            html.Td(children=[
+                                dcc.Loading(
+                                    dcc.Graph(
+                                        id="graph-tput"
+                                    ),
+                                )
+                            ], style={"width": "80%"}),
+                            html.Td(children=[
+                                dcc.Clipboard(
+                                    target_id="tput-metadata",
+                                    title="Copy",
+                                    style={"display": "inline-block"}
+                                ),
+                                html.Nobr(" "),
+                                html.Nobr(" "),
+                                dcc.Markdown(
+                                    children="**Throughput**",
+                                    style={"display": "inline-block"}
+                                ),
+                                html.Pre(
+                                    id="tput-metadata",
+                                    children="Click on data point in the graph"
+                                ),
+                                html.Div(
+                                    id="div-lat-metadata",
+                                    style=self.STYLE_HIDEN,
+                                    children=[
+                                        dcc.Clipboard(
+                                            target_id="lat-metadata",
+                                            title="Copy",
+                                            style={"display": "inline-block"}
+                                        ),
+                                        html.Nobr(" "),
+                                        html.Nobr(" "),
+                                        dcc.Markdown(
+                                            children="**Latency**",
+                                            style={"display": "inline-block"}
+                                        ),
+                                        html.Pre(
+                                            id="lat-metadata",
+                                            children= \
+                                            "Click on data point in the graph"
+                                        )
+                                    ]
+                                )
+                            ], style={"width": "20%"}),
+                        ]
+                    ),
+                    html.Tr(
+                        id="div-latency",
+                        style=self.STYLE_HIDEN,
+                        children=[
+                            html.Td(children=[
+                                dcc.Loading(
+                                    dcc.Graph(
+                                        id="graph-latency"
+                                    )
+                                )
+                            ], style={"width": "80%"}),
+                            html.Td(children=[
+                                dcc.Loading(
+                                    dcc.Graph(
+                                        id="graph-latency-hdrh",
+                                        style=self.STYLE_INLINE,
+                                        figure=self.NO_GRAPH
+                                    )
+                                )
+                            ], style={"width": "20%"}),
+                        ]
+                    ),
+                    html.Tr(
+                        id="div-download",
+                        style=self.STYLE_HIDEN,
+                        children=[
+                            html.Td(children=[
+                                dcc.Loading(
+                                    children=[
+                                        html.Button(
+                                            id="btn-download-data",
+                                            children=["Download Data"]
+                                        ),
+                                        dcc.Download(id="download-data")
+                                    ]
+                                )
+                            ], style={"width": "80%"}),
+                            html.Td(children=[
+                                html.Nobr(" ")
+                            ], style={"width": "20%"}),
+                        ]
+                    ),
+                ]),
             ],
             style={
                 "vertical-align": "top",
             ],
             style={
                 "vertical-align": "top",
-                "display": "none",
-                "width": "80%",
-                "padding": "5px"
+                "display": "inline-block",
+                "width": "80%"
             }
         )
 
             }
         )
 
@@ -195,9 +303,6 @@ class Layout:
                         )
                     ]
                 ),
                         )
                     ]
                 ),
-                # Debug output, TODO: Remove
-                html.H5("Debug output"),
-                html.Pre(id="div-ctrl-info")
             ]
         )
 
             ]
         )
 
@@ -289,11 +394,12 @@ class Layout:
                 html.Br(),
                 dcc.DatePickerRange(
                     id="dpr-period",
                 html.Br(),
                 dcc.DatePickerRange(
                     id="dpr-period",
-                    min_date_allowed=datetime(2021, 1, 1),
+                    min_date_allowed=datetime.utcnow() - timedelta(days=180),
                     max_date_allowed=datetime.utcnow(),
                     initial_visible_month=datetime.utcnow(),
                     start_date=datetime.utcnow() - timedelta(days=180),
                     max_date_allowed=datetime.utcnow(),
                     initial_visible_month=datetime.utcnow(),
                     start_date=datetime.utcnow() - timedelta(days=180),
-                    end_date=datetime.utcnow()
+                    end_date=datetime.utcnow(),
+                    display_format="D MMMM YY"
                 )
             ]
         )
                 )
             ]
         )
@@ -445,14 +551,17 @@ class Layout:
             return _sync_checklists(opt, sel, all, "cl-ctrl-testtype")
 
         @app.callback(
             return _sync_checklists(opt, sel, all, "cl-ctrl-testtype")
 
         @app.callback(
-            Output("graph", "figure"),
-            Output("div-ctrl-info", "children"),  # Debug output TODO: Remove
+            Output("graph-tput", "figure"),
+            Output("graph-latency", "figure"),
+            Output("div-tput", "style"),
+            Output("div-latency", "style"),
+            Output("div-lat-metadata", "style"),
+            Output("div-download", "style"),
             Output("selected-tests", "data"),  # Store
             Output("cl-selected", "options"),  # User selection
             Output("dd-ctrl-phy", "value"),
             Output("dd-ctrl-area", "value"),
             Output("dd-ctrl-test", "value"),
             Output("selected-tests", "data"),  # Store
             Output("cl-selected", "options"),  # User selection
             Output("dd-ctrl-phy", "value"),
             Output("dd-ctrl-area", "value"),
             Output("dd-ctrl-test", "value"),
-            Output("div-plotting-area", "style"),
             State("selected-tests", "data"),  # Store
             State("cl-selected", "value"),
             State("dd-ctrl-phy", "value"),
             State("selected-tests", "data"),  # Store
             State("cl-selected", "value"),
             State("dd-ctrl-phy", "value"),
@@ -486,29 +595,51 @@ class Layout:
                 else:
                     return list()
 
                 else:
                     return list()
 
+            class RetunValue:
+                def __init__(self) -> None:
+                    self._output = {
+                        "graph-tput-figure": no_update,
+                        "graph-lat-figure": no_update,
+                        "div-tput-style": no_update,
+                        "div-latency-style": no_update,
+                        "div-lat-metadata-style": no_update,
+                        "div-download-style": no_update,
+                        "selected-tests-data": no_update,
+                        "cl-selected-options": no_update,
+                        "dd-ctrl-phy-value": no_update,
+                        "dd-ctrl-area-value": no_update,
+                        "dd-ctrl-test-value": no_update,
+                    }
+
+                def value(self):
+                    return tuple(self._output.values())
+
+                def set_values(self, kwargs: dict) -> None:
+                    for key, val in kwargs.items():
+                        if key in self._output:
+                            self._output[key] = val
+                        else:
+                            raise KeyError(f"The key {key} is not defined.")
+
+
             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 
             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 
-            d_start = datetime(
-                 int(d_start[0:4]), int(d_start[5:7]), int(d_start[8:10])
-            )
-            d_end = datetime(
-                 int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10])
-            )
+            d_start = datetime(int(d_start[0:4]), int(d_start[5:7]),
+                int(d_start[8:10]))
+            d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10]))
+
+            output = RetunValue()
 
             if trigger_id == "btn-ctrl-add":
                 # Add selected test to the list of tests in store:
                 if phy and area and test and cores and framesizes and testtypes:
 
             if trigger_id == "btn-ctrl-add":
                 # Add selected test to the list of tests in store:
                 if phy and area and test and cores and framesizes and testtypes:
-
-                    # TODO: Add validation
-
                     if store_sel is None:
                         store_sel = list()
                     if store_sel is None:
                         store_sel = list()
-
                     for core in cores:
                         for framesize in framesizes:
                             for ttype in testtypes:
                                 tid = (
                     for core in cores:
                         for framesize in framesizes:
                             for ttype in testtypes:
                                 tid = (
-                                    f"{phy}-"
+                                    f"{phy.replace('af_xdp', 'af-xdp')}-"
                                     f"{area}-"
                                     f"{framesize.lower()}-"
                                     f"{core.lower()}-"
                                     f"{area}-"
                                     f"{framesize.lower()}-"
                                     f"{core.lower()}-"
@@ -525,13 +656,32 @@ class Layout:
                                         "core": core.lower(),
                                         "testtype": ttype.lower()
                                     })
                                         "core": core.lower(),
                                         "testtype": ttype.lower()
                                     })
-                return (no_update, no_update, store_sel, _list_tests(), None,
-                    None, None, no_update)
+                output.set_values({
+                    "selected-tests-data": store_sel,
+                    "cl-selected-options": _list_tests(),
+                    "dd-ctrl-phy-value": None,
+                    "dd-ctrl-area-value": None,
+                    "dd-ctrl-test-value": None,
+                })
 
             elif trigger_id in ("btn-sel-display", "dpr-period"):
 
             elif trigger_id in ("btn-sel-display", "dpr-period"):
-                fig, style = _update_graph(store_sel, d_start, d_end)
-                return (fig, pformat(store_sel), no_update, no_update,
-                    no_update, no_update, no_update, style)
+                fig_tput, fig_lat = graph_trending(
+                    self.data, store_sel, self.layout, d_start, d_end
+                )
+                output.set_values({
+                    "graph-tput-figure": \
+                        fig_tput if fig_tput else self.NO_GRAPH,
+                    "graph-lat-figure": \
+                        fig_lat if fig_lat else self.NO_GRAPH,
+                    "div-tput-style": \
+                        self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
+                    "div-latency-style": \
+                        self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
+                    "div-lat-metadata-style": \
+                        self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
+                    "div-download-style": \
+                        self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
+                })
 
             elif trigger_id == "btn-sel-remove":
                 if list_sel:
 
             elif trigger_id == "btn-sel-remove":
                 if list_sel:
@@ -540,91 +690,93 @@ class Layout:
                         if item["id"] not in list_sel:
                             new_store_sel.append(item)
                     store_sel = new_store_sel
                         if item["id"] not in list_sel:
                             new_store_sel.append(item)
                     store_sel = new_store_sel
-                    fig, style = _update_graph(store_sel, d_start, d_end)
-                return (fig, pformat(store_sel), store_sel, _list_tests(),
-                    no_update, no_update, no_update, style)
+                if store_sel:
+                    fig_tput, fig_lat = graph_trending(
+                        self.data, store_sel, self.layout, d_start, d_end
+                    )
+                    output.set_values({
+                        "graph-tput-figure": \
+                            fig_tput if fig_tput else self.NO_GRAPH,
+                        "graph-lat-figure": \
+                            fig_lat if fig_lat else self.NO_GRAPH,
+                        "div-tput-style": \
+                            self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
+                        "div-latency-style": \
+                            self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
+                        "div-lat-metadata-style": \
+                            self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
+                        "div-download-style": \
+                            self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
+                        "selected-tests-data": store_sel,
+                        "cl-selected-options": _list_tests()
+                    })
+                else:
+                    output.set_values({
+                        "graph-tput-figure": self.NO_GRAPH,
+                        "graph-lat-figure": self.NO_GRAPH,
+                        "div-tput-style": self.STYLE_HIDEN,
+                        "div-latency-style": self.STYLE_HIDEN,
+                        "div-lat-metadata-style": self.STYLE_HIDEN,
+                        "div-download-style": self.STYLE_HIDEN,
+                        "selected-tests-data": store_sel,
+                        "cl-selected-options": _list_tests()
+                    })
+
+            return output.value()
 
 
-        def _update_graph(sel, start, end):
+        @app.callback(
+            Output("tput-metadata", "children"),
+            Input("graph-tput", "clickData")
+        )
+        def _show_tput_metadata(hover_data):
             """
             """
             """
             """
+            if not hover_data:
+                raise PreventUpdate
 
 
-            if not sel:
-                return no_update, no_update
-
-            def _is_selected(label, sel):
-                for itm in sel:
-                    phy = itm["phy"].split("-")
-                    if len(phy) == 4:
-                        topo, arch, nic, drv = phy
-                    else:
-                        continue
-                    if nic not in label:
-                        continue
-                    if drv != "dpdk" and drv not in label:
-                        continue
-                    if itm["test"] not in label:
-                        continue
-                    if itm["framesize"] not in label:
-                        continue
-                    if itm["core"] not in label:
-                        continue
-                    if itm["testtype"] not in label:
-                        continue
-                    return (
-                        f"{itm['phy']}-{itm['framesize']}-{itm['core']}-"
-                        f"{itm['test']}-{itm['testtype']}"
-                    )
-                else:
-                    return None
+            return hover_data["points"][0]["text"].replace("<br>", "\n")
 
 
-            style={
-                "vertical-align": "top",
-                "display": "inline-block",
-                "width": "80%",
-                "padding": "5px"
-            }
+        @app.callback(
+            Output("graph-latency-hdrh", "figure"),
+            Output("graph-latency-hdrh", "style"),
+            Output("lat-metadata", "children"),
+            Input("graph-latency", "clickData")
+        )
+        def _show_latency_hdhr(hover_data):
+            """
+            """
+            if not hover_data:
+                raise PreventUpdate
 
 
-            fig = go.Figure()
-            dates = self.data.iloc[[0], 1:].values.flatten().tolist()[::-1]
-            x_data = [
-                datetime(
-                    int(date[0:4]), int(date[4:6]), int(date[6:8]),
-                    int(date[9:11]), int(date[12:])
-                ) for date in dates
-            ]
-            x_data_range = [
-                date for date in x_data if date >= start and date <= end
-            ]
-            vpp = self.data.iloc[[1], 1:].values.flatten().tolist()[::-1]
-            csit = list(self.data.columns[1:])[::-1]
-            labels = list(self.data["Build Number:"][3:])
-            for i in range(3, len(self.data)):
-                name = _is_selected(labels[i-3], sel)
-                if not name:
+            graph = no_update
+            hdrh_data = hover_data["points"][0].get("customdata", None)
+            if hdrh_data:
+                graph = graph_hdrh_latency(hdrh_data, self.layout)
+
+            return (
+                graph,
+                self.STYLE_INLINE,
+                hover_data["points"][0]["text"].replace("<br>", "\n")
+            )
+
+        @app.callback(
+            Output("download-data", "data"),
+            State("selected-tests", "data"),
+            Input("btn-download-data", "n_clicks"),
+            prevent_initial_call=True
+        )
+        def _download_data(store_sel, n_clicks):
+            """
+            """
+
+            if not n_clicks:
+                raise PreventUpdate
+
+            df = pd.DataFrame()
+            for itm in store_sel:
+                sel_data = select_trending_data(self.data, itm)
+                if sel_data is None:
                     continue
                     continue
-                y_data = [
-                    float(v) / 1e6 for v in \
-                        self.data.iloc[[i], 1:].values.flatten().tolist()[::-1]
-                ]
-                hover_txt = list()
-                for x_idx, x_itm in enumerate(x_data):
-                    hover_txt.append(
-                        f"date: {x_itm}<br>"
-                        f"average [Mpps]: {y_data[x_idx]}<br>"
-                        f"vpp-ref: {vpp[x_idx]}<br>"
-                        f"csit-ref: {csit[x_idx]}"
-                    )
-                fig.add_trace(
-                    go.Scatter(
-                        x=x_data_range,
-                        y= y_data,
-                        name=name,
-                        mode="markers+lines",
-                        text=hover_txt,
-                        hoverinfo=u"text+name"
-                    )
-                )
-            layout = self._graph_layout.get("plot-trending", dict())
-            fig.update_layout(layout)
+                df = pd.concat([df, sel_data], ignore_index=True)
 
 
-            return fig, style
+            return dcc.send_data_frame(df.to_csv, "trending_data.csv")
Integration tests