UTI: Improvements in detailed hover metadata information
[csit.git] / resources / tools / dash / app / pal / trending / layout.py
1 # Copyright (c) 2022 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Plotly Dash HTML layout override.
15 """
16
17 import pandas as pd
18
19 from dash import dcc
20 from dash import html
21 from dash import callback_context, no_update
22 from dash import Input, Output, State, callback
23 from dash.exceptions import PreventUpdate
24 from yaml import load, FullLoader, YAMLError
25 from datetime import datetime, timedelta
26
27 from ..data.data import Data
28 from .graphs import graph_trending, graph_hdrh_latency, \
29     select_trending_data
30
31
32 class Layout:
33     """
34     """
35
36     STYLE_HIDEN = {"display": "none"}
37     STYLE_BLOCK = {"display": "block", "vertical-align": "top"}
38     STYLE_INLINE ={
39         "display": "inline-block",
40         "vertical-align": "top",
41         "width": "33%"
42     }
43     NO_GRAPH = {"data": [], "layout": {}, "frames": []}
44
45     def __init__(self, app, html_layout_file, spec_file, graph_layout_file,
46         data_spec_file):
47         """
48         """
49
50         # Inputs
51         self._app = app
52         self._html_layout_file = html_layout_file
53         self._spec_file = spec_file
54         self._graph_layout_file = graph_layout_file
55         self._data_spec_file = data_spec_file
56
57         # Read the data:
58         data_mrr = Data(
59             data_spec_file=self._data_spec_file,
60             debug=True
61         ).read_trending_mrr()
62
63         data_ndrpdr = Data(
64             data_spec_file=self._data_spec_file,
65             debug=True
66         ).read_trending_ndrpdr()
67
68         self._data = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
69
70         # Read from files:
71         self._html_layout = ""
72         self._spec_tbs = None
73         self._graph_layout = None
74
75         try:
76             with open(self._html_layout_file, "r") as file_read:
77                 self._html_layout = file_read.read()
78         except IOError as err:
79             raise RuntimeError(
80                 f"Not possible to open the file {self._html_layout_file}\n{err}"
81             )
82
83         try:
84             with open(self._spec_file, "r") as file_read:
85                 self._spec_tbs = load(file_read, Loader=FullLoader)
86         except IOError as err:
87             raise RuntimeError(
88                 f"Not possible to open the file {self._spec_file,}\n{err}"
89             )
90         except YAMLError as err:
91             raise RuntimeError(
92                 f"An error occurred while parsing the specification file "
93                 f"{self._spec_file,}\n"
94                 f"{err}"
95             )
96
97         try:
98             with open(self._graph_layout_file, "r") as file_read:
99                 self._graph_layout = load(file_read, Loader=FullLoader)
100         except IOError as err:
101             raise RuntimeError(
102                 f"Not possible to open the file {self._graph_layout_file}\n"
103                 f"{err}"
104             )
105         except YAMLError as err:
106             raise RuntimeError(
107                 f"An error occurred while parsing the specification file "
108                 f"{self._graph_layout_file}\n"
109                 f"{err}"
110             )
111
112         # Callbacks:
113         if self._app is not None and hasattr(self, 'callbacks'):
114             self.callbacks(self._app)
115
116     @property
117     def html_layout(self):
118         return self._html_layout
119
120     @property
121     def spec_tbs(self):
122         return self._spec_tbs
123
124     @property
125     def data(self):
126         return self._data
127
128     @property
129     def layout(self):
130         return self._graph_layout
131
132     def add_content(self):
133         """
134         """
135         if self.html_layout and self.spec_tbs:
136             return html.Div(
137                 id="div-main",
138                 children=[
139                     dcc.Store(id="selected-tests"),
140                     self._add_ctrl_div(),
141                     self._add_plotting_div()
142                 ]
143             )
144         else:
145             return html.Div(
146             id="div-main-error",
147             children="An Error Occured."
148         )
149
150     def _add_ctrl_div(self):
151         """Add div with controls. It is placed on the left side.
152         """
153         return html.Div(
154             id="div-controls",
155             children=[
156                 html.Div(
157                     id="div-controls-tabs",
158                     children=[
159                         self._add_ctrl_select(),
160                         self._add_ctrl_shown()
161                     ]
162                 )
163             ],
164             style={
165                 "display": "inline-block",
166                 "width": "18%",
167                 "padding": "5px"
168             }
169         )
170
171     def _add_plotting_div(self):
172         """Add div with plots and tables. It is placed on the right side.
173         """
174         return html.Div(
175             id="div-plotting-area",
176             children=[
177                 dcc.Loading(
178                     id="loading-graph",
179                     children=[
180                         dcc.Graph(
181                             id="graph-tput",
182                             style=self.STYLE_HIDEN
183                         ),
184                         dcc.Graph(
185                             id="graph-latency",
186                             style=self.STYLE_HIDEN
187                         )
188                     ],
189                     type="circle"
190                 ),
191                 html.Div(
192                     children=[
193                         html.Div(
194                             id="div-tput-metadata",
195                             children=[
196                                 html.Button(
197                                     id="btn-download-data",
198                                     children=["Download Data"],
199                                     style={"display": "block"}
200                                 ),
201                                 dcc.Download(id="download-data"),
202                                 dcc.Clipboard(
203                                     target_id="tput-metadata",
204                                     title="Copy",
205                                     style={"display": "inline-block"}
206                                 ),
207                                 html.Nobr(" "),
208                                 html.Nobr(" "),
209                                 dcc.Markdown(
210                                     children="**Throughput**",
211                                     style={"display": "inline-block"}
212                                 ),
213                                 html.Pre(
214                                     id="tput-metadata",
215                                     children="Click on data points in the graph"
216                                 )
217                             ],
218                             style=self.STYLE_HIDEN
219                         ),
220                         html.Div(
221                             id="div-latency-metadata",
222                             children=[
223                                 dcc.Clipboard(
224                                     target_id="latency-metadata",
225                                     title="Copy",
226                                     style={"display": "inline-block"}
227                                 ),
228                                 html.Nobr(" "),
229                                 html.Nobr(" "),
230                                 dcc.Markdown(
231                                     children="**Latency**",
232                                     style={"display": "inline-block"}
233                                 ),
234                                 html.Pre(
235                                     id="latency-metadata",
236                                     children="Click on data points in the graph"
237                                 )
238                             ],
239                             style=self.STYLE_HIDEN
240                         ),
241                         html.Div(
242                             id="div-latency-hdrh",
243                             children=[
244                                 dcc.Loading(
245                                     id="loading-hdrh-latency-graph",
246                                     children=[
247                                         dcc.Graph(
248                                             id="graph-latency-hdrh"
249                                         )
250                                     ],
251                                     type="circle"
252                                 )
253                             ],
254                             style=self.STYLE_HIDEN
255                         )
256                     ]
257                 )
258             ],
259             style={
260                 "vertical-align": "top",
261                 "display": "inline-block",
262                 "width": "80%",
263                 "padding": "5px"
264             }
265         )
266
267     def _add_ctrl_shown(self):
268         """
269         """
270         return html.Div(
271             id="div-ctrl-shown",
272             children=[
273                 html.H5("Selected tests"),
274                 html.Div(
275                     id="container-selected-tests",
276                     children=[
277                         dcc.Checklist(
278                             id="cl-selected",
279                             options=[],
280                             labelStyle={"display": "block"}
281                         ),
282                         html.Button(
283                             id="btn-sel-remove",
284                             children="Remove Selected",
285                             disabled=False
286                         ),
287                         html.Button(
288                             id="btn-sel-display",
289                             children="Display",
290                             disabled=False
291                         )
292                     ]
293                 ),
294             ]
295         )
296
297     def _add_ctrl_select(self):
298         """
299         """
300         return html.Div(
301             id="div-ctrl-select",
302             children=[
303                 html.H5("Physical Test Bed Topology, NIC and Driver"),
304                 dcc.Dropdown(
305                     id="dd-ctrl-phy",
306                     placeholder="Select a Physical Test Bed Topology...",
307                     multi=False,
308                     clearable=False,
309                     options=[
310                         {"label": k, "value": k} for k in self.spec_tbs.keys()
311                     ],
312                 ),
313                 html.H5("Area"),
314                 dcc.Dropdown(
315                     id="dd-ctrl-area",
316                     placeholder="Select an Area...",
317                     disabled=True,
318                     multi=False,
319                     clearable=False,
320                 ),
321                 html.H5("Test"),
322                 dcc.Dropdown(
323                     id="dd-ctrl-test",
324                     placeholder="Select a Test...",
325                     disabled=True,
326                     multi=False,
327                     clearable=False,
328                 ),
329                 html.Div(
330                     id="div-ctrl-core",
331                     children=[
332                         html.H5("Number of Cores"),
333                         dcc.Checklist(
334                             id="cl-ctrl-core-all",
335                             options=[{"label": "All", "value": "all"}, ],
336                             labelStyle={"display": "inline-block"}
337                         ),
338                         dcc.Checklist(
339                             id="cl-ctrl-core",
340                             labelStyle={"display": "inline-block"}
341                         )
342                     ],
343                     style={"display": "none"}
344                 ),
345                 html.Div(
346                     id="div-ctrl-framesize",
347                     children=[
348                         html.H5("Frame Size"),
349                         dcc.Checklist(
350                             id="cl-ctrl-framesize-all",
351                             options=[{"label": "All", "value": "all"}, ],
352                             labelStyle={"display": "inline-block"}
353                         ),
354                         dcc.Checklist(
355                             id="cl-ctrl-framesize",
356                             labelStyle={"display": "inline-block"}
357                         )
358                     ],
359                     style={"display": "none"}
360                 ),
361                 html.Div(
362                     id="div-ctrl-testtype",
363                     children=[
364                         html.H5("Test Type"),
365                         dcc.Checklist(
366                             id="cl-ctrl-testtype-all",
367                             options=[{"label": "All", "value": "all"}, ],
368                             labelStyle={"display": "inline-block"}
369                         ),
370                         dcc.Checklist(
371                             id="cl-ctrl-testtype",
372                             labelStyle={"display": "inline-block"}
373                         )
374                     ],
375                     style={"display": "none"}
376                 ),
377                 html.Button(
378                     id="btn-ctrl-add",
379                     children="Add",
380                     disabled=True
381                 ),
382                 html.Br(),
383                 dcc.DatePickerRange(
384                     id="dpr-period",
385                     min_date_allowed=datetime.utcnow() - timedelta(days=180),
386                     max_date_allowed=datetime.utcnow(),
387                     initial_visible_month=datetime.utcnow(),
388                     start_date=datetime.utcnow() - timedelta(days=180),
389                     end_date=datetime.utcnow(),
390                     display_format="D MMMM YY"
391                 )
392             ]
393         )
394
395     def callbacks(self, app):
396
397         @app.callback(
398             Output("dd-ctrl-area", "options"),
399             Output("dd-ctrl-area", "disabled"),
400             Input("dd-ctrl-phy", "value"),
401         )
402         def _update_dd_area(phy):
403             """
404             """
405
406             if phy is None:
407                 raise PreventUpdate
408
409             try:
410                 options = [
411                     {"label": self.spec_tbs[phy][v]["label"], "value": v}
412                         for v in [v for v in self.spec_tbs[phy].keys()]
413                 ]
414                 disable = False
415             except KeyError:
416                 options = list()
417                 disable = True
418
419             return options, disable
420
421         @app.callback(
422             Output("dd-ctrl-test", "options"),
423             Output("dd-ctrl-test", "disabled"),
424             State("dd-ctrl-phy", "value"),
425             Input("dd-ctrl-area", "value"),
426         )
427         def _update_dd_test(phy, area):
428             """
429             """
430
431             if not area:
432                 raise PreventUpdate
433
434             try:
435                 options = [
436                     {"label": v, "value": v}
437                         for v in self.spec_tbs[phy][area]["test"]
438                 ]
439                 disable = False
440             except KeyError:
441                 options = list()
442                 disable = True
443
444             return options, disable
445
446         @app.callback(
447             Output("div-ctrl-core", "style"),
448             Output("cl-ctrl-core", "options"),
449             Output("div-ctrl-framesize", "style"),
450             Output("cl-ctrl-framesize", "options"),
451             Output("div-ctrl-testtype", "style"),
452             Output("cl-ctrl-testtype", "options"),
453             Output("btn-ctrl-add", "disabled"),
454             State("dd-ctrl-phy", "value"),
455             State("dd-ctrl-area", "value"),
456             Input("dd-ctrl-test", "value"),
457         )
458         def _update_btn_add(phy, area, test):
459             """
460             """
461
462             if test is None:
463                 raise PreventUpdate
464
465             core_style = {"display": "none"}
466             core_opts = []
467             framesize_style = {"display": "none"}
468             framesize_opts = []
469             testtype_style = {"display": "none"}
470             testtype_opts = []
471             add_disabled = True
472             if phy and area and test:
473                 core_style = {"display": "block"}
474                 core_opts = [
475                     {"label": v, "value": v}
476                         for v in self.spec_tbs[phy][area]["core"]
477                 ]
478                 framesize_style = {"display": "block"}
479                 framesize_opts = [
480                     {"label": v, "value": v}
481                         for v in self.spec_tbs[phy][area]["frame-size"]
482                 ]
483                 testtype_style = {"display": "block"}
484                 testtype_opts = [
485                     {"label": v, "value": v}
486                         for v in self.spec_tbs[phy][area]["test-type"]
487                 ]
488                 add_disabled = False
489
490             return (
491                 core_style, core_opts,
492                 framesize_style, framesize_opts,
493                 testtype_style, testtype_opts,
494                 add_disabled
495             )
496
497         def _sync_checklists(opt, sel, all, id):
498             """
499             """
500             options = {v["value"] for v in opt}
501             input_id = callback_context.triggered[0]["prop_id"].split(".")[0]
502             if input_id == id:
503                 all = ["all"] if set(sel) == options else list()
504             else:
505                 sel = list(options) if all else list()
506             return sel, all
507
508         @app.callback(
509             Output("cl-ctrl-core", "value"),
510             Output("cl-ctrl-core-all", "value"),
511             State("cl-ctrl-core", "options"),
512             Input("cl-ctrl-core", "value"),
513             Input("cl-ctrl-core-all", "value"),
514             prevent_initial_call=True
515         )
516         def _sync_cl_core(opt, sel, all):
517             return _sync_checklists(opt, sel, all, "cl-ctrl-core")
518
519         @app.callback(
520             Output("cl-ctrl-framesize", "value"),
521             Output("cl-ctrl-framesize-all", "value"),
522             State("cl-ctrl-framesize", "options"),
523             Input("cl-ctrl-framesize", "value"),
524             Input("cl-ctrl-framesize-all", "value"),
525             prevent_initial_call=True
526         )
527         def _sync_cl_framesize(opt, sel, all):
528             return _sync_checklists(opt, sel, all, "cl-ctrl-framesize")
529
530         @app.callback(
531             Output("cl-ctrl-testtype", "value"),
532             Output("cl-ctrl-testtype-all", "value"),
533             State("cl-ctrl-testtype", "options"),
534             Input("cl-ctrl-testtype", "value"),
535             Input("cl-ctrl-testtype-all", "value"),
536             prevent_initial_call=True
537         )
538         def _sync_cl_testtype(opt, sel, all):
539             return _sync_checklists(opt, sel, all, "cl-ctrl-testtype")
540
541         @app.callback(
542             Output("graph-tput", "figure"),
543             Output("graph-tput", "style"),
544             Output("div-tput-metadata", "style"),
545             Output("graph-latency", "figure"),
546             Output("graph-latency", "style"),
547             Output("div-latency-metadata", "style"),
548             Output("selected-tests", "data"),  # Store
549             Output("cl-selected", "options"),  # User selection
550             Output("dd-ctrl-phy", "value"),
551             Output("dd-ctrl-area", "value"),
552             Output("dd-ctrl-test", "value"),
553             State("selected-tests", "data"),  # Store
554             State("cl-selected", "value"),
555             State("dd-ctrl-phy", "value"),
556             State("dd-ctrl-area", "value"),
557             State("dd-ctrl-test", "value"),
558             State("cl-ctrl-core", "value"),
559             State("cl-ctrl-framesize", "value"),
560             State("cl-ctrl-testtype", "value"),
561             Input("btn-ctrl-add", "n_clicks"),
562             Input("btn-sel-display", "n_clicks"),
563             Input("btn-sel-remove", "n_clicks"),
564             Input("dpr-period", "start_date"),
565             Input("dpr-period", "end_date"),
566             prevent_initial_call=True
567         )
568         def _process_list(store_sel, list_sel, phy, area, test, cores,
569                 framesizes, testtypes, btn_add, btn_display, btn_remove,
570                 d_start, d_end):
571             """
572             """
573
574             if not (btn_add or btn_display or btn_remove or d_start or d_end):
575                 raise PreventUpdate
576
577             def _list_tests():
578                 # Display selected tests with checkboxes:
579                 if store_sel:
580                     return [
581                         {"label": v["id"], "value": v["id"]} for v in store_sel
582                     ]
583                 else:
584                     return list()
585
586             class RetunValue:
587                 def __init__(self) -> None:
588                     self._output = {
589                         "graph-tput-figure": no_update,
590                         "graph-tput-style": no_update,
591                         "div-tput-metadata-style": no_update,
592                         "graph-lat-figure": no_update,
593                         "graph-lat-style": no_update,
594                         "div-lat-metadata-style": no_update,
595                         "selected-tests-data": no_update,
596                         "cl-selected-options": no_update,
597                         "dd-ctrl-phy-value": no_update,
598                         "dd-ctrl-area-value": no_update,
599                         "dd-ctrl-test-value": no_update,
600                     }
601
602                 def value(self):
603                     return tuple(self._output.values())
604
605                 def set_values(self, kwargs: dict) -> None:
606                     for key, val in kwargs.items():
607                         if key in self._output:
608                             self._output[key] = val
609                         else:
610                             raise KeyError(f"The key {key} is not defined.")
611
612
613             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
614
615             d_start = datetime(int(d_start[0:4]), int(d_start[5:7]),
616                 int(d_start[8:10]))
617             d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10]))
618
619             output = RetunValue()
620
621             if trigger_id == "btn-ctrl-add":
622                 # Add selected test to the list of tests in store:
623                 if phy and area and test and cores and framesizes and testtypes:
624                     if store_sel is None:
625                         store_sel = list()
626                     for core in cores:
627                         for framesize in framesizes:
628                             for ttype in testtypes:
629                                 tid = (
630                                     f"{phy.replace('af_xdp', 'af-xdp')}-"
631                                     f"{area}-"
632                                     f"{framesize.lower()}-"
633                                     f"{core.lower()}-"
634                                     f"{test}-"
635                                     f"{ttype.lower()}"
636                                 )
637                                 if tid not in [itm["id"] for itm in store_sel]:
638                                     store_sel.append({
639                                         "id": tid,
640                                         "phy": phy,
641                                         "area": area,
642                                         "test": test,
643                                         "framesize": framesize.lower(),
644                                         "core": core.lower(),
645                                         "testtype": ttype.lower()
646                                     })
647                 output.set_values({
648                     "selected-tests-data": store_sel,
649                     "cl-selected-options": _list_tests(),
650                     "dd-ctrl-phy-value": None,
651                     "dd-ctrl-area-value": None,
652                     "dd-ctrl-test-value": None,
653                 })
654
655             elif trigger_id in ("btn-sel-display", "dpr-period"):
656                 fig_tput, fig_lat = graph_trending(
657                     self.data, store_sel, self.layout, d_start, d_end
658                 )
659                 output.set_values({
660                     "graph-tput-figure": \
661                         fig_tput if fig_tput else self.NO_GRAPH,
662                     "graph-tput-style": \
663                         self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
664                     "div-tput-metadata-style": \
665                         self.STYLE_INLINE if fig_tput else self.STYLE_HIDEN,
666                     "graph-lat-figure": \
667                         fig_lat if fig_lat else self.NO_GRAPH,
668                     "graph-lat-style": \
669                         self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
670                     "div-lat-metadata-style": \
671                         self.STYLE_INLINE if fig_lat else self.STYLE_HIDEN
672                 })
673
674             elif trigger_id == "btn-sel-remove":
675                 if list_sel:
676                     new_store_sel = list()
677                     for item in store_sel:
678                         if item["id"] not in list_sel:
679                             new_store_sel.append(item)
680                     store_sel = new_store_sel
681                 if store_sel:
682                     fig_tput, fig_lat = graph_trending(
683                         self.data, store_sel, self.layout, d_start, d_end
684                     )
685                     output.set_values({
686                         "graph-tput-figure": \
687                             fig_tput if fig_tput else self.NO_GRAPH,
688                         "graph-tput-style": \
689                             self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
690                         "div-tput-metadata-style": \
691                             self.STYLE_INLINE if fig_tput else self.STYLE_HIDEN,
692                         "graph-lat-figure": \
693                             fig_lat if fig_lat else self.NO_GRAPH,
694                         "graph-lat-style": \
695                             self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
696                         "div-lat-metadata-style": \
697                             self.STYLE_INLINE if fig_lat else self.STYLE_HIDEN,
698                         "selected-tests-data": store_sel,
699                         "cl-selected-options": _list_tests()
700                     })
701                 else:
702                     output.set_values({
703                         "graph-tput-figure": self.NO_GRAPH,
704                         "graph-tput-style": self.STYLE_HIDEN,
705                         "div-tput-metadata-style": self.STYLE_HIDEN,
706                         "graph-lat-figure": self.NO_GRAPH,
707                         "graph-lat-style": self.STYLE_HIDEN,
708                         "div-lat-metadata-style": self.STYLE_HIDEN,
709                         "selected-tests-data": store_sel,
710                         "cl-selected-options": _list_tests()
711                     })
712
713             return output.value()
714
715         @app.callback(
716             Output("tput-metadata", "children"),
717             Input("graph-tput", "clickData")
718         )
719         def _show_tput_metadata(hover_data):
720             """
721             """
722             if not hover_data:
723                 raise PreventUpdate
724
725             return hover_data["points"][0]["text"].replace("<br>", "\n"),
726
727         @app.callback(
728             Output("latency-metadata", "children"),
729             Output("graph-latency-hdrh", "figure"),
730             Output("div-latency-hdrh", "style"),
731             Input("graph-latency", "clickData")
732         )
733         def _show_latency_metadata(hover_data):
734             """
735             """
736             if not hover_data:
737                 raise PreventUpdate
738
739             graph = no_update
740             hdrh_data = hover_data["points"][0].get("customdata", None)
741             if hdrh_data:
742                 graph = graph_hdrh_latency(hdrh_data, self.layout)
743
744             return (
745                 hover_data["points"][0]["text"].replace("<br>", "\n"),
746                 graph,
747                 self.STYLE_INLINE if graph else self.STYLE_HIDEN
748             )
749
750         @app.callback(
751             Output("download-data", "data"),
752             State("selected-tests", "data"),
753             Input("btn-download-data", "n_clicks"),
754             prevent_initial_call=True
755         )
756         def _download_data(store_sel, n_clicks):
757             """
758             """
759
760             if not n_clicks:
761                 raise PreventUpdate
762
763             df = pd.DataFrame()
764             for itm in store_sel:
765                 sel_data = select_trending_data(self.data, itm)
766                 if sel_data is None:
767                     continue
768                 df = pd.concat([df, sel_data], ignore_index=True)
769
770             return dcc.send_data_frame(df.to_csv, "trending_data.csv")