X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fdash%2Fapp%2Fpal%2Ftrending%2Flayout.py;h=66af2f0de24a5fd14b6a1b05034c0ff62e7d488e;hp=69ce0c416dae4175a1b39c1819d43fb9909cce31;hb=45615cddd926540756f19328cad7078cfc9a1219;hpb=b2cb835b34c7404b2aaee3ec30700c67537da66d diff --git a/resources/tools/dash/app/pal/trending/layout.py b/resources/tools/dash/app/pal/trending/layout.py index 69ce0c416d..66af2f0de2 100644 --- a/resources/tools/dash/app/pal/trending/layout.py +++ b/resources/tools/dash/app/pal/trending/layout.py @@ -14,20 +14,36 @@ """Plotly Dash HTML layout override. """ -import logging +import pandas as pd from dash import dcc from dash import html -from dash import Input, Output, callback +from dash import callback_context, no_update +from dash import Input, Output, State from dash.exceptions import PreventUpdate +import dash_bootstrap_components as dbc from yaml import load, FullLoader, YAMLError +from datetime import datetime, timedelta + +from ..data.data import Data +from .graphs import graph_trending, graph_hdrh_latency, \ + select_trending_data class Layout: """ """ - def __init__(self, app, html_layout_file, spec_file): + STYLE_HIDEN = {"display": "none"} + STYLE_BLOCK = {"display": "block", "vertical-align": "top"} + STYLE_INLINE ={ + "display": "inline-block", + "vertical-align": "top" + } + NO_GRAPH = {"data": [], "layout": {}, "frames": []} + + def __init__(self, app, html_layout_file, spec_file, graph_layout_file, + data_spec_file): """ """ @@ -35,26 +51,61 @@ class Layout: self._app = app self._html_layout_file = html_layout_file self._spec_file = spec_file + self._graph_layout_file = graph_layout_file + self._data_spec_file = data_spec_file + + # Read the data: + data_mrr = Data( + data_spec_file=self._data_spec_file, + debug=True + ).read_trending_mrr() + + data_ndrpdr = Data( + data_spec_file=self._data_spec_file, + debug=True + ).read_trending_ndrpdr() + + self._data = pd.concat([data_mrr, data_ndrpdr], ignore_index=True) # Read from files: self._html_layout = "" - self._spec_test = None + self._spec_tbs = None + self._graph_layout = None try: - with open(self._html_layout_file, "r") as layout_file: - self._html_layout = layout_file.read() + with open(self._html_layout_file, "r") as file_read: + self._html_layout = file_read.read() except IOError as err: - logging.error(f"Not possible to open the file {layout_file}\n{err}") + raise RuntimeError( + f"Not possible to open the file {self._html_layout_file}\n{err}" + ) try: with open(self._spec_file, "r") as file_read: - self._spec_test = load(file_read, Loader=FullLoader) + self._spec_tbs = load(file_read, Loader=FullLoader) + except IOError as err: + raise RuntimeError( + f"Not possible to open the file {self._spec_file,}\n{err}" + ) + except YAMLError as err: + raise RuntimeError( + f"An error occurred while parsing the specification file " + f"{self._spec_file,}\n" + f"{err}" + ) + + try: + with open(self._graph_layout_file, "r") as file_read: + self._graph_layout = load(file_read, Loader=FullLoader) except IOError as err: - logging.error(f"Not possible to open the file {spec_file}\n{err}") + raise RuntimeError( + f"Not possible to open the file {self._graph_layout_file}\n" + f"{err}" + ) except YAMLError as err: - logging.error( + raise RuntimeError( f"An error occurred while parsing the specification file " - f"{spec_file}\n" + f"{self._graph_layout_file}\n" f"{err}" ) @@ -62,41 +113,31 @@ class Layout: if self._app is not None and hasattr(self, 'callbacks'): self.callbacks(self._app) - # User choice (one test): - self._test_selection = { - "phy": "", - "area": "", - "test": "", - "core": "", - "frame-size": "", - "test-type": "" - } - @property def html_layout(self): return self._html_layout @property - def spec_test(self): - return self._spec_test - - def _reset_test_selection(self): - self._test_selection = { - "phy": "", - "area": "", - "test": "", - "core": "", - "frame-size": "", - "test-type": "" - } + def spec_tbs(self): + return self._spec_tbs + + @property + def data(self): + return self._data + + @property + def layout(self): + return self._graph_layout def add_content(self): """ """ - if self._html_layout and self._spec_test: + if self.html_layout and self.spec_tbs: return html.Div( id="div-main", children=[ + dcc.Store(id="selected-tests"), + self._add_navbar(), self._add_ctrl_div(), self._add_plotting_div() ] @@ -107,6 +148,22 @@ class Layout: children="An Error Occured." ) + def _add_navbar(self): + """Add nav element with navigation panel. It is placed on the top. + """ + return dbc.NavbarSimple( + children=[ + dbc.NavItem( + dbc.NavLink("Continuous Performance Trending", href="#") + ) + ], + brand="Dashboard", + brand_href="/", + color="dark", + dark=True, + fluid=True, + ) + def _add_ctrl_div(self): """Add div with controls. It is placed on the left side. """ @@ -123,8 +180,7 @@ class Layout: ], style={ "display": "inline-block", - "width": "18%", - "padding": "5px" + "width": "20%" } ) @@ -134,21 +190,107 @@ class Layout: return html.Div( id="div-plotting-area", children=[ - # Only a visible note. - # TODO: Add content. - html.H3( - "Graphs and Tables", - style={ - "vertical-align": "middle", - "text-align": "center" - } - ) + html.Table(children=[ + html.Tr( + id="div-tput", + style=self.STYLE_HIDEN, + children=[ + html.Td(children=[ + dcc.Loading( + dcc.Graph( + id="graph-tput" + ), + ) + ], style={"width": "80%"}), + html.Td(children=[ + dcc.Clipboard( + target_id="tput-metadata", + title="Copy", + style={"display": "inline-block"} + ), + html.Nobr(" "), + html.Nobr(" "), + dcc.Markdown( + children="**Throughput**", + style={"display": "inline-block"} + ), + html.Pre( + id="tput-metadata", + children="Click on data point in the graph" + ), + html.Div( + id="div-lat-metadata", + style=self.STYLE_HIDEN, + children=[ + dcc.Clipboard( + target_id="lat-metadata", + title="Copy", + style={"display": "inline-block"} + ), + html.Nobr(" "), + html.Nobr(" "), + dcc.Markdown( + children="**Latency**", + style={"display": "inline-block"} + ), + html.Pre( + id="lat-metadata", + children= \ + "Click on data point in the graph" + ) + ] + ) + ], style={"width": "20%"}), + ] + ), + html.Tr( + id="div-latency", + style=self.STYLE_HIDEN, + children=[ + html.Td(children=[ + dcc.Loading( + dcc.Graph( + id="graph-latency" + ) + ) + ], style={"width": "80%"}), + html.Td(children=[ + dcc.Loading( + dcc.Graph( + id="graph-latency-hdrh", + style=self.STYLE_INLINE, + figure=self.NO_GRAPH + ) + ) + ], style={"width": "20%"}), + ] + ), + html.Tr( + id="div-download", + style=self.STYLE_HIDEN, + children=[ + html.Td(children=[ + dcc.Loading( + children=[ + html.Button( + id="btn-download-data", + children=["Download Data"] + ), + dcc.Download(id="download-data") + ] + ) + ], style={"width": "80%"}), + html.Td(children=[ + html.Nobr(" ") + ], style={"width": "20%"}), + ] + ), + ]), ], style={ - "vertical-align": "middle", + "vertical-align": "top", "display": "inline-block", - "width": "80%", - "padding": "5px" + "width": "80%" } ) @@ -157,7 +299,29 @@ class Layout: """ return html.Div( id="div-ctrl-shown", - children="List of selected tests" + children=[ + html.H5("Selected tests"), + html.Div( + id="container-selected-tests", + children=[ + dcc.Checklist( + id="cl-selected", + options=[], + labelStyle={"display": "block"} + ), + html.Button( + id="btn-sel-remove", + children="Remove Selected", + disabled=False + ), + html.Button( + id="btn-sel-display", + children="Display", + disabled=False + ) + ] + ), + ] ) def _add_ctrl_select(self): @@ -166,23 +330,17 @@ class Layout: return html.Div( id="div-ctrl-select", children=[ - html.Br(), - html.Div( - children="Physical Test Bed Topology, NIC and Driver" - ), + html.H5("Physical Test Bed Topology, NIC and Driver"), dcc.Dropdown( id="dd-ctrl-phy", placeholder="Select a Physical Test Bed Topology...", multi=False, clearable=False, options=[ - {"label": k, "value": k} for k in self._spec_test.keys() + {"label": k, "value": k} for k in self.spec_tbs.keys() ], ), - html.Br(), - html.Div( - children="Area" - ), + html.H5("Area"), dcc.Dropdown( id="dd-ctrl-area", placeholder="Select an Area...", @@ -190,10 +348,7 @@ class Layout: multi=False, clearable=False, ), - html.Br(), - html.Div( - children="Test" - ), + html.H5("Test"), dcc.Dropdown( id="dd-ctrl-test", placeholder="Select a Test...", @@ -201,50 +356,68 @@ class Layout: multi=False, clearable=False, ), - - # Change to radio buttons: - html.Br(), html.Div( - children="Number of Cores" - ), - dcc.Dropdown( - id="dd-ctrl-core", - placeholder="Select a Number of Cores...", - disabled=True, - multi=False, - clearable=False, + id="div-ctrl-core", + children=[ + html.H5("Number of Cores"), + dcc.Checklist( + id="cl-ctrl-core-all", + options=[{"label": "All", "value": "all"}, ], + labelStyle={"display": "inline-block"} + ), + dcc.Checklist( + id="cl-ctrl-core", + labelStyle={"display": "inline-block"} + ) + ], + style={"display": "none"} ), - html.Br(), html.Div( - children="Frame Size" - ), - dcc.Dropdown( - id="dd-ctrl-framesize", - placeholder="Select a Frame Size...", - disabled=True, - multi=False, - clearable=False, + id="div-ctrl-framesize", + children=[ + html.H5("Frame Size"), + dcc.Checklist( + id="cl-ctrl-framesize-all", + options=[{"label": "All", "value": "all"}, ], + labelStyle={"display": "inline-block"} + ), + dcc.Checklist( + id="cl-ctrl-framesize", + labelStyle={"display": "inline-block"} + ) + ], + style={"display": "none"} ), - html.Br(), html.Div( - children="Test Type" - ), - dcc.Dropdown( - id="dd-ctrl-testtype", - placeholder="Select a Test Type...", - disabled=True, - multi=False, - clearable=False, + id="div-ctrl-testtype", + children=[ + html.H5("Test Type"), + dcc.Checklist( + id="cl-ctrl-testtype-all", + options=[{"label": "All", "value": "all"}, ], + labelStyle={"display": "inline-block"} + ), + dcc.Checklist( + id="cl-ctrl-testtype", + labelStyle={"display": "inline-block"} + ) + ], + style={"display": "none"} ), - html.Br(), html.Button( id="btn-ctrl-add", children="Add", disabled=True ), html.Br(), - html.Div( - id="div-ctrl-info" + dcc.DatePickerRange( + id="dpr-period", + min_date_allowed=datetime.utcnow() - timedelta(days=180), + max_date_allowed=datetime.utcnow(), + initial_visible_month=datetime.utcnow(), + start_date=datetime.utcnow() - timedelta(days=180), + end_date=datetime.utcnow(), + display_format="D MMMM YY" ) ] ) @@ -265,85 +438,363 @@ class Layout: try: options = [ - {"label": self._spec_test[phy][v]["label"], "value": v} - for v in [v for v in self._spec_test[phy].keys()] + {"label": self.spec_tbs[phy][v]["label"], "value": v} + for v in [v for v in self.spec_tbs[phy].keys()] ] + disable = False except KeyError: options = list() + disable = True - return options, False + return options, disable @app.callback( Output("dd-ctrl-test", "options"), Output("dd-ctrl-test", "disabled"), - Input("dd-ctrl-phy", "value"), + State("dd-ctrl-phy", "value"), Input("dd-ctrl-area", "value"), ) def _update_dd_test(phy, area): """ """ - if not all((phy, area, )): + if not area: raise PreventUpdate try: options = [ {"label": v, "value": v} - for v in self._spec_test[phy][area]["test"] + for v in self.spec_tbs[phy][area]["test"] ] + disable = False except KeyError: options = list() + disable = True - return options, False + return options, disable @app.callback( + Output("div-ctrl-core", "style"), + Output("cl-ctrl-core", "options"), + Output("div-ctrl-framesize", "style"), + Output("cl-ctrl-framesize", "options"), + Output("div-ctrl-testtype", "style"), + Output("cl-ctrl-testtype", "options"), Output("btn-ctrl-add", "disabled"), - Input("dd-ctrl-phy", "value"), - Input("dd-ctrl-area", "value"), + State("dd-ctrl-phy", "value"), + State("dd-ctrl-area", "value"), Input("dd-ctrl-test", "value"), ) def _update_btn_add(phy, area, test): """ """ - if all((phy, area, test, )): - self._test_selection["phy"] = phy - self._test_selection["area"] = area - self._test_selection["test"] = test - return False + if test is None: + raise PreventUpdate + + core_style = {"display": "none"} + core_opts = [] + framesize_style = {"display": "none"} + framesize_opts = [] + testtype_style = {"display": "none"} + testtype_opts = [] + add_disabled = True + if phy and area and test: + core_style = {"display": "block"} + core_opts = [ + {"label": v, "value": v} + for v in self.spec_tbs[phy][area]["core"] + ] + framesize_style = {"display": "block"} + framesize_opts = [ + {"label": v, "value": v} + for v in self.spec_tbs[phy][area]["frame-size"] + ] + testtype_style = {"display": "block"} + testtype_opts = [ + {"label": v, "value": v} + for v in self.spec_tbs[phy][area]["test-type"] + ] + add_disabled = False + + return ( + core_style, core_opts, + framesize_style, framesize_opts, + testtype_style, testtype_opts, + add_disabled + ) + + def _sync_checklists(opt, sel, all, id): + """ + """ + options = {v["value"] for v in opt} + input_id = callback_context.triggered[0]["prop_id"].split(".")[0] + if input_id == id: + all = ["all"] if set(sel) == options else list() else: - return True + sel = list(options) if all else list() + return sel, all + + @app.callback( + Output("cl-ctrl-core", "value"), + Output("cl-ctrl-core-all", "value"), + State("cl-ctrl-core", "options"), + Input("cl-ctrl-core", "value"), + Input("cl-ctrl-core-all", "value"), + prevent_initial_call=True + ) + def _sync_cl_core(opt, sel, all): + return _sync_checklists(opt, sel, all, "cl-ctrl-core") + + @app.callback( + Output("cl-ctrl-framesize", "value"), + Output("cl-ctrl-framesize-all", "value"), + State("cl-ctrl-framesize", "options"), + Input("cl-ctrl-framesize", "value"), + Input("cl-ctrl-framesize-all", "value"), + prevent_initial_call=True + ) + def _sync_cl_framesize(opt, sel, all): + return _sync_checklists(opt, sel, all, "cl-ctrl-framesize") @app.callback( - Output("div-ctrl-info", "children"), + Output("cl-ctrl-testtype", "value"), + Output("cl-ctrl-testtype-all", "value"), + State("cl-ctrl-testtype", "options"), + Input("cl-ctrl-testtype", "value"), + Input("cl-ctrl-testtype-all", "value"), + prevent_initial_call=True + ) + def _sync_cl_testtype(opt, sel, all): + return _sync_checklists(opt, sel, all, "cl-ctrl-testtype") + + @app.callback( + Output("graph-tput", "figure"), + Output("graph-latency", "figure"), + Output("div-tput", "style"), + Output("div-latency", "style"), + Output("div-lat-metadata", "style"), + Output("div-download", "style"), + Output("selected-tests", "data"), # Store + Output("cl-selected", "options"), # User selection Output("dd-ctrl-phy", "value"), Output("dd-ctrl-area", "value"), Output("dd-ctrl-test", "value"), - Output("btn-ctrl-add", "n_clicks"), - Input("btn-ctrl-add", "n_clicks") + State("selected-tests", "data"), # Store + State("cl-selected", "value"), + State("dd-ctrl-phy", "value"), + State("dd-ctrl-area", "value"), + State("dd-ctrl-test", "value"), + State("cl-ctrl-core", "value"), + State("cl-ctrl-framesize", "value"), + State("cl-ctrl-testtype", "value"), + Input("btn-ctrl-add", "n_clicks"), + Input("btn-sel-display", "n_clicks"), + Input("btn-sel-remove", "n_clicks"), + Input("dpr-period", "start_date"), + Input("dpr-period", "end_date"), + prevent_initial_call=True ) - def _print_user_selection(n_clicks): + def _process_list(store_sel, list_sel, phy, area, test, cores, + framesizes, testtypes, btn_add, btn_display, btn_remove, + d_start, d_end): """ """ - logging.info(f"\n\n{n_clicks}\n\n") + if not (btn_add or btn_display or btn_remove or d_start or d_end): + raise PreventUpdate + + def _list_tests(): + # Display selected tests with checkboxes: + if store_sel: + return [ + {"label": v["id"], "value": v["id"]} for v in store_sel + ] + else: + return list() + + class RetunValue: + def __init__(self) -> None: + self._output = { + "graph-tput-figure": no_update, + "graph-lat-figure": no_update, + "div-tput-style": no_update, + "div-latency-style": no_update, + "div-lat-metadata-style": no_update, + "div-download-style": no_update, + "selected-tests-data": no_update, + "cl-selected-options": no_update, + "dd-ctrl-phy-value": no_update, + "dd-ctrl-area-value": no_update, + "dd-ctrl-test-value": no_update, + } - if not n_clicks: + def value(self): + return tuple(self._output.values()) + + def set_values(self, kwargs: dict) -> None: + for key, val in kwargs.items(): + if key in self._output: + self._output[key] = val + else: + raise KeyError(f"The key {key} is not defined.") + + + trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0] + + d_start = datetime(int(d_start[0:4]), int(d_start[5:7]), + int(d_start[8:10])) + d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10])) + + output = RetunValue() + + if trigger_id == "btn-ctrl-add": + # Add selected test to the list of tests in store: + if phy and area and test and cores and framesizes and testtypes: + if store_sel is None: + store_sel = list() + for core in cores: + for framesize in framesizes: + for ttype in testtypes: + tid = ( + f"{phy.replace('af_xdp', 'af-xdp')}-" + f"{area}-" + f"{framesize.lower()}-" + f"{core.lower()}-" + f"{test}-" + f"{ttype.lower()}" + ) + if tid not in [itm["id"] for itm in store_sel]: + store_sel.append({ + "id": tid, + "phy": phy, + "area": area, + "test": test, + "framesize": framesize.lower(), + "core": core.lower(), + "testtype": ttype.lower() + }) + output.set_values({ + "selected-tests-data": store_sel, + "cl-selected-options": _list_tests(), + "dd-ctrl-phy-value": None, + "dd-ctrl-area-value": None, + "dd-ctrl-test-value": None, + }) + + elif trigger_id in ("btn-sel-display", "dpr-period"): + fig_tput, fig_lat = graph_trending( + self.data, store_sel, self.layout, d_start, d_end + ) + output.set_values({ + "graph-tput-figure": \ + fig_tput if fig_tput else self.NO_GRAPH, + "graph-lat-figure": \ + fig_lat if fig_lat else self.NO_GRAPH, + "div-tput-style": \ + self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN, + "div-latency-style": \ + self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN, + "div-lat-metadata-style": \ + self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN, + "div-download-style": \ + self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN, + }) + + elif trigger_id == "btn-sel-remove": + if list_sel: + new_store_sel = list() + for item in store_sel: + if item["id"] not in list_sel: + new_store_sel.append(item) + store_sel = new_store_sel + if store_sel: + fig_tput, fig_lat = graph_trending( + self.data, store_sel, self.layout, d_start, d_end + ) + output.set_values({ + "graph-tput-figure": \ + fig_tput if fig_tput else self.NO_GRAPH, + "graph-lat-figure": \ + fig_lat if fig_lat else self.NO_GRAPH, + "div-tput-style": \ + self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN, + "div-latency-style": \ + self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN, + "div-lat-metadata-style": \ + self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN, + "div-download-style": \ + self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN, + "selected-tests-data": store_sel, + "cl-selected-options": _list_tests() + }) + else: + output.set_values({ + "graph-tput-figure": self.NO_GRAPH, + "graph-lat-figure": self.NO_GRAPH, + "div-tput-style": self.STYLE_HIDEN, + "div-latency-style": self.STYLE_HIDEN, + "div-lat-metadata-style": self.STYLE_HIDEN, + "div-download-style": self.STYLE_HIDEN, + "selected-tests-data": store_sel, + "cl-selected-options": _list_tests() + }) + + return output.value() + + @app.callback( + Output("tput-metadata", "children"), + Input("graph-tput", "clickData") + ) + def _show_tput_metadata(hover_data): + """ + """ + if not hover_data: raise PreventUpdate - selected = ( - f"{self._test_selection['phy']} # " - f"{self._test_selection['area']} # " - f"{self._test_selection['test']} # " - f"{n_clicks}\n" - ) + return hover_data["points"][0]["text"].replace("
", "\n") - self._reset_test_selection() + @app.callback( + Output("graph-latency-hdrh", "figure"), + Output("graph-latency-hdrh", "style"), + Output("lat-metadata", "children"), + Input("graph-latency", "clickData") + ) + def _show_latency_hdhr(hover_data): + """ + """ + if not hover_data: + raise PreventUpdate + + graph = no_update + hdrh_data = hover_data["points"][0].get("customdata", None) + if hdrh_data: + graph = graph_hdrh_latency(hdrh_data, self.layout) return ( - selected, - None, - None, - None, - 0, + graph, + self.STYLE_INLINE, + hover_data["points"][0]["text"].replace("
", "\n") ) + + @app.callback( + Output("download-data", "data"), + State("selected-tests", "data"), + Input("btn-download-data", "n_clicks"), + prevent_initial_call=True + ) + def _download_data(store_sel, n_clicks): + """ + """ + + if not n_clicks: + raise PreventUpdate + + df = pd.DataFrame() + for itm in store_sel: + sel_data = select_trending_data(self.data, itm) + if sel_data is None: + continue + df = pd.concat([df, sel_data], ignore_index=True) + + return dcc.send_data_frame(df.to_csv, "trending_data.csv")