afa459bf76f5df906318c6894273a0907d841695
[csit.git] / resources / tools / dash / app / pal / trending / layout.py
1 # Copyright (c) 2022 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Plotly Dash HTML layout override.
15 """
16
17 import pandas as pd
18
19 from dash import dcc
20 from dash import html
21 from dash import callback_context, no_update
22 from dash import Input, Output, State
23 from dash.exceptions import PreventUpdate
24 from yaml import load, FullLoader, YAMLError
25 from datetime import datetime, timedelta
26
27 from ..data.data import Data
28 from .graphs import graph_trending, graph_hdrh_latency, \
29     select_trending_data
30
31
32 class Layout:
33     """
34     """
35
36     STYLE_HIDEN = {"display": "none"}
37     STYLE_BLOCK = {"display": "block", "vertical-align": "top"}
38     STYLE_INLINE ={
39         "display": "inline-block",
40         "vertical-align": "top"
41     }
42     NO_GRAPH = {"data": [], "layout": {}, "frames": []}
43
44     def __init__(self, app, html_layout_file, spec_file, graph_layout_file,
45         data_spec_file):
46         """
47         """
48
49         # Inputs
50         self._app = app
51         self._html_layout_file = html_layout_file
52         self._spec_file = spec_file
53         self._graph_layout_file = graph_layout_file
54         self._data_spec_file = data_spec_file
55
56         # Read the data:
57         data_mrr = Data(
58             data_spec_file=self._data_spec_file,
59             debug=True
60         ).read_trending_mrr()
61
62         data_ndrpdr = Data(
63             data_spec_file=self._data_spec_file,
64             debug=True
65         ).read_trending_ndrpdr()
66
67         self._data = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
68
69         # Read from files:
70         self._html_layout = ""
71         self._spec_tbs = None
72         self._graph_layout = None
73
74         try:
75             with open(self._html_layout_file, "r") as file_read:
76                 self._html_layout = file_read.read()
77         except IOError as err:
78             raise RuntimeError(
79                 f"Not possible to open the file {self._html_layout_file}\n{err}"
80             )
81
82         try:
83             with open(self._spec_file, "r") as file_read:
84                 self._spec_tbs = load(file_read, Loader=FullLoader)
85         except IOError as err:
86             raise RuntimeError(
87                 f"Not possible to open the file {self._spec_file,}\n{err}"
88             )
89         except YAMLError as err:
90             raise RuntimeError(
91                 f"An error occurred while parsing the specification file "
92                 f"{self._spec_file,}\n"
93                 f"{err}"
94             )
95
96         try:
97             with open(self._graph_layout_file, "r") as file_read:
98                 self._graph_layout = load(file_read, Loader=FullLoader)
99         except IOError as err:
100             raise RuntimeError(
101                 f"Not possible to open the file {self._graph_layout_file}\n"
102                 f"{err}"
103             )
104         except YAMLError as err:
105             raise RuntimeError(
106                 f"An error occurred while parsing the specification file "
107                 f"{self._graph_layout_file}\n"
108                 f"{err}"
109             )
110
111         # Callbacks:
112         if self._app is not None and hasattr(self, 'callbacks'):
113             self.callbacks(self._app)
114
115     @property
116     def html_layout(self):
117         return self._html_layout
118
119     @property
120     def spec_tbs(self):
121         return self._spec_tbs
122
123     @property
124     def data(self):
125         return self._data
126
127     @property
128     def layout(self):
129         return self._graph_layout
130
131     def add_content(self):
132         """
133         """
134         if self.html_layout and self.spec_tbs:
135             return html.Div(
136                 id="div-main",
137                 children=[
138                     dcc.Store(id="selected-tests"),
139                     self._add_ctrl_div(),
140                     self._add_plotting_div()
141                 ]
142             )
143         else:
144             return html.Div(
145             id="div-main-error",
146             children="An Error Occured."
147         )
148
149     def _add_ctrl_div(self):
150         """Add div with controls. It is placed on the left side.
151         """
152         return html.Div(
153             id="div-controls",
154             children=[
155                 html.Div(
156                     id="div-controls-tabs",
157                     children=[
158                         self._add_ctrl_select(),
159                         self._add_ctrl_shown()
160                     ]
161                 )
162             ],
163             style={
164                 "display": "inline-block",
165                 "width": "20%"
166             }
167         )
168
169     def _add_plotting_div(self):
170         """Add div with plots and tables. It is placed on the right side.
171         """
172         return html.Div(
173             id="div-plotting-area",
174             children=[
175                 html.Table(children=[
176                     html.Tr(
177                         id="div-tput",
178                         style=self.STYLE_HIDEN,
179                         children=[
180                             html.Td(children=[
181                                 dcc.Loading(
182                                     dcc.Graph(
183                                         id="graph-tput"
184                                     ),
185                                 )
186                             ], style={"width": "80%"}),
187                             html.Td(children=[
188                                 dcc.Clipboard(
189                                     target_id="tput-metadata",
190                                     title="Copy",
191                                     style={"display": "inline-block"}
192                                 ),
193                                 html.Nobr(" "),
194                                 html.Nobr(" "),
195                                 dcc.Markdown(
196                                     children="**Throughput**",
197                                     style={"display": "inline-block"}
198                                 ),
199                                 html.Pre(
200                                     id="tput-metadata",
201                                     children="Click on data point in the graph"
202                                 ),
203                                 html.Div(
204                                     id="div-lat-metadata",
205                                     style=self.STYLE_HIDEN,
206                                     children=[
207                                         dcc.Clipboard(
208                                             target_id="lat-metadata",
209                                             title="Copy",
210                                             style={"display": "inline-block"}
211                                         ),
212                                         html.Nobr(" "),
213                                         html.Nobr(" "),
214                                         dcc.Markdown(
215                                             children="**Latency**",
216                                             style={"display": "inline-block"}
217                                         ),
218                                         html.Pre(
219                                             id="lat-metadata",
220                                             children= \
221                                             "Click on data point in the graph"
222                                         )
223                                     ]
224                                 )
225                             ], style={"width": "20%"}),
226                         ]
227                     ),
228                     html.Tr(
229                         id="div-latency",
230                         style=self.STYLE_HIDEN,
231                         children=[
232                             html.Td(children=[
233                                 dcc.Loading(
234                                     dcc.Graph(
235                                         id="graph-latency"
236                                     )
237                                 )
238                             ], style={"width": "80%"}),
239                             html.Td(children=[
240                                 dcc.Loading(
241                                     dcc.Graph(
242                                         id="graph-latency-hdrh",
243                                         style=self.STYLE_INLINE,
244                                         figure=self.NO_GRAPH
245                                     )
246                                 )
247                             ], style={"width": "20%"}),
248                         ]
249                     ),
250                     html.Tr(
251                         id="div-download",
252                         style=self.STYLE_HIDEN,
253                         children=[
254                             html.Td(children=[
255                                 dcc.Loading(
256                                     children=[
257                                         html.Button(
258                                             id="btn-download-data",
259                                             children=["Download Data"]
260                                         ),
261                                         dcc.Download(id="download-data")
262                                     ]
263                                 )
264                             ], style={"width": "80%"}),
265                             html.Td(children=[
266                                 html.Nobr(" ")
267                             ], style={"width": "20%"}),
268                         ]
269                     ),
270                 ]),
271             ],
272             style={
273                 "vertical-align": "top",
274                 "display": "inline-block",
275                 "width": "80%"
276             }
277         )
278
279     def _add_ctrl_shown(self):
280         """
281         """
282         return html.Div(
283             id="div-ctrl-shown",
284             children=[
285                 html.H5("Selected tests"),
286                 html.Div(
287                     id="container-selected-tests",
288                     children=[
289                         dcc.Checklist(
290                             id="cl-selected",
291                             options=[],
292                             labelStyle={"display": "block"}
293                         ),
294                         html.Button(
295                             id="btn-sel-remove",
296                             children="Remove Selected",
297                             disabled=False
298                         ),
299                         html.Button(
300                             id="btn-sel-display",
301                             children="Display",
302                             disabled=False
303                         )
304                     ]
305                 ),
306             ]
307         )
308
309     def _add_ctrl_select(self):
310         """
311         """
312         return html.Div(
313             id="div-ctrl-select",
314             children=[
315                 html.H5("Physical Test Bed Topology, NIC and Driver"),
316                 dcc.Dropdown(
317                     id="dd-ctrl-phy",
318                     placeholder="Select a Physical Test Bed Topology...",
319                     multi=False,
320                     clearable=False,
321                     options=[
322                         {"label": k, "value": k} for k in self.spec_tbs.keys()
323                     ],
324                 ),
325                 html.H5("Area"),
326                 dcc.Dropdown(
327                     id="dd-ctrl-area",
328                     placeholder="Select an Area...",
329                     disabled=True,
330                     multi=False,
331                     clearable=False,
332                 ),
333                 html.H5("Test"),
334                 dcc.Dropdown(
335                     id="dd-ctrl-test",
336                     placeholder="Select a Test...",
337                     disabled=True,
338                     multi=False,
339                     clearable=False,
340                 ),
341                 html.Div(
342                     id="div-ctrl-core",
343                     children=[
344                         html.H5("Number of Cores"),
345                         dcc.Checklist(
346                             id="cl-ctrl-core-all",
347                             options=[{"label": "All", "value": "all"}, ],
348                             labelStyle={"display": "inline-block"}
349                         ),
350                         dcc.Checklist(
351                             id="cl-ctrl-core",
352                             labelStyle={"display": "inline-block"}
353                         )
354                     ],
355                     style={"display": "none"}
356                 ),
357                 html.Div(
358                     id="div-ctrl-framesize",
359                     children=[
360                         html.H5("Frame Size"),
361                         dcc.Checklist(
362                             id="cl-ctrl-framesize-all",
363                             options=[{"label": "All", "value": "all"}, ],
364                             labelStyle={"display": "inline-block"}
365                         ),
366                         dcc.Checklist(
367                             id="cl-ctrl-framesize",
368                             labelStyle={"display": "inline-block"}
369                         )
370                     ],
371                     style={"display": "none"}
372                 ),
373                 html.Div(
374                     id="div-ctrl-testtype",
375                     children=[
376                         html.H5("Test Type"),
377                         dcc.Checklist(
378                             id="cl-ctrl-testtype-all",
379                             options=[{"label": "All", "value": "all"}, ],
380                             labelStyle={"display": "inline-block"}
381                         ),
382                         dcc.Checklist(
383                             id="cl-ctrl-testtype",
384                             labelStyle={"display": "inline-block"}
385                         )
386                     ],
387                     style={"display": "none"}
388                 ),
389                 html.Button(
390                     id="btn-ctrl-add",
391                     children="Add",
392                     disabled=True
393                 ),
394                 html.Br(),
395                 dcc.DatePickerRange(
396                     id="dpr-period",
397                     min_date_allowed=datetime.utcnow() - timedelta(days=180),
398                     max_date_allowed=datetime.utcnow(),
399                     initial_visible_month=datetime.utcnow(),
400                     start_date=datetime.utcnow() - timedelta(days=180),
401                     end_date=datetime.utcnow(),
402                     display_format="D MMMM YY"
403                 )
404             ]
405         )
406
407     def callbacks(self, app):
408
409         @app.callback(
410             Output("dd-ctrl-area", "options"),
411             Output("dd-ctrl-area", "disabled"),
412             Input("dd-ctrl-phy", "value"),
413         )
414         def _update_dd_area(phy):
415             """
416             """
417
418             if phy is None:
419                 raise PreventUpdate
420
421             try:
422                 options = [
423                     {"label": self.spec_tbs[phy][v]["label"], "value": v}
424                         for v in [v for v in self.spec_tbs[phy].keys()]
425                 ]
426                 disable = False
427             except KeyError:
428                 options = list()
429                 disable = True
430
431             return options, disable
432
433         @app.callback(
434             Output("dd-ctrl-test", "options"),
435             Output("dd-ctrl-test", "disabled"),
436             State("dd-ctrl-phy", "value"),
437             Input("dd-ctrl-area", "value"),
438         )
439         def _update_dd_test(phy, area):
440             """
441             """
442
443             if not area:
444                 raise PreventUpdate
445
446             try:
447                 options = [
448                     {"label": v, "value": v}
449                         for v in self.spec_tbs[phy][area]["test"]
450                 ]
451                 disable = False
452             except KeyError:
453                 options = list()
454                 disable = True
455
456             return options, disable
457
458         @app.callback(
459             Output("div-ctrl-core", "style"),
460             Output("cl-ctrl-core", "options"),
461             Output("div-ctrl-framesize", "style"),
462             Output("cl-ctrl-framesize", "options"),
463             Output("div-ctrl-testtype", "style"),
464             Output("cl-ctrl-testtype", "options"),
465             Output("btn-ctrl-add", "disabled"),
466             State("dd-ctrl-phy", "value"),
467             State("dd-ctrl-area", "value"),
468             Input("dd-ctrl-test", "value"),
469         )
470         def _update_btn_add(phy, area, test):
471             """
472             """
473
474             if test is None:
475                 raise PreventUpdate
476
477             core_style = {"display": "none"}
478             core_opts = []
479             framesize_style = {"display": "none"}
480             framesize_opts = []
481             testtype_style = {"display": "none"}
482             testtype_opts = []
483             add_disabled = True
484             if phy and area and test:
485                 core_style = {"display": "block"}
486                 core_opts = [
487                     {"label": v, "value": v}
488                         for v in self.spec_tbs[phy][area]["core"]
489                 ]
490                 framesize_style = {"display": "block"}
491                 framesize_opts = [
492                     {"label": v, "value": v}
493                         for v in self.spec_tbs[phy][area]["frame-size"]
494                 ]
495                 testtype_style = {"display": "block"}
496                 testtype_opts = [
497                     {"label": v, "value": v}
498                         for v in self.spec_tbs[phy][area]["test-type"]
499                 ]
500                 add_disabled = False
501
502             return (
503                 core_style, core_opts,
504                 framesize_style, framesize_opts,
505                 testtype_style, testtype_opts,
506                 add_disabled
507             )
508
509         def _sync_checklists(opt, sel, all, id):
510             """
511             """
512             options = {v["value"] for v in opt}
513             input_id = callback_context.triggered[0]["prop_id"].split(".")[0]
514             if input_id == id:
515                 all = ["all"] if set(sel) == options else list()
516             else:
517                 sel = list(options) if all else list()
518             return sel, all
519
520         @app.callback(
521             Output("cl-ctrl-core", "value"),
522             Output("cl-ctrl-core-all", "value"),
523             State("cl-ctrl-core", "options"),
524             Input("cl-ctrl-core", "value"),
525             Input("cl-ctrl-core-all", "value"),
526             prevent_initial_call=True
527         )
528         def _sync_cl_core(opt, sel, all):
529             return _sync_checklists(opt, sel, all, "cl-ctrl-core")
530
531         @app.callback(
532             Output("cl-ctrl-framesize", "value"),
533             Output("cl-ctrl-framesize-all", "value"),
534             State("cl-ctrl-framesize", "options"),
535             Input("cl-ctrl-framesize", "value"),
536             Input("cl-ctrl-framesize-all", "value"),
537             prevent_initial_call=True
538         )
539         def _sync_cl_framesize(opt, sel, all):
540             return _sync_checklists(opt, sel, all, "cl-ctrl-framesize")
541
542         @app.callback(
543             Output("cl-ctrl-testtype", "value"),
544             Output("cl-ctrl-testtype-all", "value"),
545             State("cl-ctrl-testtype", "options"),
546             Input("cl-ctrl-testtype", "value"),
547             Input("cl-ctrl-testtype-all", "value"),
548             prevent_initial_call=True
549         )
550         def _sync_cl_testtype(opt, sel, all):
551             return _sync_checklists(opt, sel, all, "cl-ctrl-testtype")
552
553         @app.callback(
554             Output("graph-tput", "figure"),
555             Output("graph-latency", "figure"),
556             Output("div-tput", "style"),
557             Output("div-latency", "style"),
558             Output("div-lat-metadata", "style"),
559             Output("div-download", "style"),
560             Output("selected-tests", "data"),  # Store
561             Output("cl-selected", "options"),  # User selection
562             Output("dd-ctrl-phy", "value"),
563             Output("dd-ctrl-area", "value"),
564             Output("dd-ctrl-test", "value"),
565             State("selected-tests", "data"),  # Store
566             State("cl-selected", "value"),
567             State("dd-ctrl-phy", "value"),
568             State("dd-ctrl-area", "value"),
569             State("dd-ctrl-test", "value"),
570             State("cl-ctrl-core", "value"),
571             State("cl-ctrl-framesize", "value"),
572             State("cl-ctrl-testtype", "value"),
573             Input("btn-ctrl-add", "n_clicks"),
574             Input("btn-sel-display", "n_clicks"),
575             Input("btn-sel-remove", "n_clicks"),
576             Input("dpr-period", "start_date"),
577             Input("dpr-period", "end_date"),
578             prevent_initial_call=True
579         )
580         def _process_list(store_sel, list_sel, phy, area, test, cores,
581                 framesizes, testtypes, btn_add, btn_display, btn_remove,
582                 d_start, d_end):
583             """
584             """
585
586             if not (btn_add or btn_display or btn_remove or d_start or d_end):
587                 raise PreventUpdate
588
589             def _list_tests():
590                 # Display selected tests with checkboxes:
591                 if store_sel:
592                     return [
593                         {"label": v["id"], "value": v["id"]} for v in store_sel
594                     ]
595                 else:
596                     return list()
597
598             class RetunValue:
599                 def __init__(self) -> None:
600                     self._output = {
601                         "graph-tput-figure": no_update,
602                         "graph-lat-figure": no_update,
603                         "div-tput-style": no_update,
604                         "div-latency-style": no_update,
605                         "div-lat-metadata-style": no_update,
606                         "div-download-style": no_update,
607                         "selected-tests-data": no_update,
608                         "cl-selected-options": no_update,
609                         "dd-ctrl-phy-value": no_update,
610                         "dd-ctrl-area-value": no_update,
611                         "dd-ctrl-test-value": no_update,
612                     }
613
614                 def value(self):
615                     return tuple(self._output.values())
616
617                 def set_values(self, kwargs: dict) -> None:
618                     for key, val in kwargs.items():
619                         if key in self._output:
620                             self._output[key] = val
621                         else:
622                             raise KeyError(f"The key {key} is not defined.")
623
624
625             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
626
627             d_start = datetime(int(d_start[0:4]), int(d_start[5:7]),
628                 int(d_start[8:10]))
629             d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10]))
630
631             output = RetunValue()
632
633             if trigger_id == "btn-ctrl-add":
634                 # Add selected test to the list of tests in store:
635                 if phy and area and test and cores and framesizes and testtypes:
636                     if store_sel is None:
637                         store_sel = list()
638                     for core in cores:
639                         for framesize in framesizes:
640                             for ttype in testtypes:
641                                 tid = (
642                                     f"{phy.replace('af_xdp', 'af-xdp')}-"
643                                     f"{area}-"
644                                     f"{framesize.lower()}-"
645                                     f"{core.lower()}-"
646                                     f"{test}-"
647                                     f"{ttype.lower()}"
648                                 )
649                                 if tid not in [itm["id"] for itm in store_sel]:
650                                     store_sel.append({
651                                         "id": tid,
652                                         "phy": phy,
653                                         "area": area,
654                                         "test": test,
655                                         "framesize": framesize.lower(),
656                                         "core": core.lower(),
657                                         "testtype": ttype.lower()
658                                     })
659                 output.set_values({
660                     "selected-tests-data": store_sel,
661                     "cl-selected-options": _list_tests(),
662                     "dd-ctrl-phy-value": None,
663                     "dd-ctrl-area-value": None,
664                     "dd-ctrl-test-value": None,
665                 })
666
667             elif trigger_id in ("btn-sel-display", "dpr-period"):
668                 fig_tput, fig_lat = graph_trending(
669                     self.data, store_sel, self.layout, d_start, d_end
670                 )
671                 output.set_values({
672                     "graph-tput-figure": \
673                         fig_tput if fig_tput else self.NO_GRAPH,
674                     "graph-lat-figure": \
675                         fig_lat if fig_lat else self.NO_GRAPH,
676                     "div-tput-style": \
677                         self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
678                     "div-latency-style": \
679                         self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
680                     "div-lat-metadata-style": \
681                         self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
682                     "div-download-style": \
683                         self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
684                 })
685
686             elif trigger_id == "btn-sel-remove":
687                 if list_sel:
688                     new_store_sel = list()
689                     for item in store_sel:
690                         if item["id"] not in list_sel:
691                             new_store_sel.append(item)
692                     store_sel = new_store_sel
693                 if store_sel:
694                     fig_tput, fig_lat = graph_trending(
695                         self.data, store_sel, self.layout, d_start, d_end
696                     )
697                     output.set_values({
698                         "graph-tput-figure": \
699                             fig_tput if fig_tput else self.NO_GRAPH,
700                         "graph-lat-figure": \
701                             fig_lat if fig_lat else self.NO_GRAPH,
702                         "div-tput-style": \
703                             self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
704                         "div-latency-style": \
705                             self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
706                         "div-lat-metadata-style": \
707                             self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
708                         "div-download-style": \
709                             self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
710                         "selected-tests-data": store_sel,
711                         "cl-selected-options": _list_tests()
712                     })
713                 else:
714                     output.set_values({
715                         "graph-tput-figure": self.NO_GRAPH,
716                         "graph-lat-figure": self.NO_GRAPH,
717                         "div-tput-style": self.STYLE_HIDEN,
718                         "div-latency-style": self.STYLE_HIDEN,
719                         "div-lat-metadata-style": self.STYLE_HIDEN,
720                         "div-download-style": self.STYLE_HIDEN,
721                         "selected-tests-data": store_sel,
722                         "cl-selected-options": _list_tests()
723                     })
724
725             return output.value()
726
727         @app.callback(
728             Output("tput-metadata", "children"),
729             Input("graph-tput", "clickData")
730         )
731         def _show_tput_metadata(hover_data):
732             """
733             """
734             if not hover_data:
735                 raise PreventUpdate
736
737             return hover_data["points"][0]["text"].replace("<br>", "\n")
738
739         @app.callback(
740             Output("graph-latency-hdrh", "figure"),
741             Output("graph-latency-hdrh", "style"),
742             Output("lat-metadata", "children"),
743             Input("graph-latency", "clickData")
744         )
745         def _show_latency_hdhr(hover_data):
746             """
747             """
748             if not hover_data:
749                 raise PreventUpdate
750
751             graph = no_update
752             hdrh_data = hover_data["points"][0].get("customdata", None)
753             if hdrh_data:
754                 graph = graph_hdrh_latency(hdrh_data, self.layout)
755
756             return (
757                 graph,
758                 self.STYLE_INLINE,
759                 hover_data["points"][0]["text"].replace("<br>", "\n")
760             )
761
762         @app.callback(
763             Output("download-data", "data"),
764             State("selected-tests", "data"),
765             Input("btn-download-data", "n_clicks"),
766             prevent_initial_call=True
767         )
768         def _download_data(store_sel, n_clicks):
769             """
770             """
771
772             if not n_clicks:
773                 raise PreventUpdate
774
775             df = pd.DataFrame()
776             for itm in store_sel:
777                 sel_data = select_trending_data(self.data, itm)
778                 if sel_data is None:
779                     continue
780                 df = pd.concat([df, sel_data], ignore_index=True)
781
782             return dcc.send_data_frame(df.to_csv, "trending_data.csv")