9f246d8e73211ecb16b0841bf57520d06e1397a7
[csit.git] / resources / tools / dash / app / pal / trending / layout.py
1 # Copyright (c) 2022 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Plotly Dash HTML layout override.
15 """
16
17 import pandas as pd
18
19 from dash import dcc
20 from dash import html
21 from dash import callback_context, no_update
22 from dash import Input, Output, State
23 from dash.exceptions import PreventUpdate
24 import dash_bootstrap_components as dbc
25 from yaml import load, FullLoader, YAMLError
26 from datetime import datetime, timedelta
27
28 from ..data.data import Data
29 from .graphs import graph_trending, graph_hdrh_latency, \
30     select_trending_data
31
32
33 class Layout:
34     """
35     """
36
37     NO_GRAPH = {"data": [], "layout": {}, "frames": []}
38
39     def __init__(self, app, html_layout_file, spec_file, graph_layout_file,
40         data_spec_file):
41         """
42         """
43
44         # Inputs
45         self._app = app
46         self._html_layout_file = html_layout_file
47         self._spec_file = spec_file
48         self._graph_layout_file = graph_layout_file
49         self._data_spec_file = data_spec_file
50
51         # Read the data:
52         data_mrr = Data(
53             data_spec_file=self._data_spec_file,
54             debug=True
55         ).read_trending_mrr(days=5)
56
57         data_ndrpdr = Data(
58             data_spec_file=self._data_spec_file,
59             debug=True
60         ).read_trending_ndrpdr(days=14)
61
62         self._data = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
63
64         # Read from files:
65         self._html_layout = ""
66         self._spec_tbs = None
67         self._graph_layout = None
68
69         try:
70             with open(self._html_layout_file, "r") as file_read:
71                 self._html_layout = file_read.read()
72         except IOError as err:
73             raise RuntimeError(
74                 f"Not possible to open the file {self._html_layout_file}\n{err}"
75             )
76
77         try:
78             with open(self._spec_file, "r") as file_read:
79                 self._spec_tbs = load(file_read, Loader=FullLoader)
80         except IOError as err:
81             raise RuntimeError(
82                 f"Not possible to open the file {self._spec_file,}\n{err}"
83             )
84         except YAMLError as err:
85             raise RuntimeError(
86                 f"An error occurred while parsing the specification file "
87                 f"{self._spec_file,}\n"
88                 f"{err}"
89             )
90
91         try:
92             with open(self._graph_layout_file, "r") as file_read:
93                 self._graph_layout = load(file_read, Loader=FullLoader)
94         except IOError as err:
95             raise RuntimeError(
96                 f"Not possible to open the file {self._graph_layout_file}\n"
97                 f"{err}"
98             )
99         except YAMLError as err:
100             raise RuntimeError(
101                 f"An error occurred while parsing the specification file "
102                 f"{self._graph_layout_file}\n"
103                 f"{err}"
104             )
105
106         # Callbacks:
107         if self._app is not None and hasattr(self, 'callbacks'):
108             self.callbacks(self._app)
109
110     @property
111     def html_layout(self):
112         return self._html_layout
113
114     @property
115     def spec_tbs(self):
116         return self._spec_tbs
117
118     @property
119     def data(self):
120         return self._data
121
122     @property
123     def layout(self):
124         return self._graph_layout
125
126     def add_content(self):
127         """
128         """
129         if self.html_layout and self.spec_tbs:
130             return html.Div(
131                 id="div-main",
132                 children=[
133                     dbc.Row(
134                         id="row-navbar",
135                         class_name="g-0",
136                         children=[
137                             self._add_navbar(),
138                         ]
139                     ),
140                     dcc.Loading(
141                         dbc.Offcanvas(
142                             id="offcanvas-metadata",
143                             title="Throughput And Latency",
144                             placement="end",
145                             is_open=False,
146                             children=[
147                                 dbc.Row(id="metadata-tput-lat"),
148                                 dbc.Row(id="metadata-hdrh-graph"),
149                             ]
150                         )
151                     ),
152                     dbc.Row(
153                         id="row-main",
154                         class_name="g-0 p-2",
155                         children=[
156                             dcc.Store(
157                                 id="selected-tests"
158                             ),
159                             self._add_ctrl_col(),
160                             self._add_plotting_col(),
161                         ]
162                     )
163                 ]
164             )
165         else:
166             return html.Div(
167                 id="div-main-error",
168                 children=[
169                     dbc.Alert(
170                         [
171                             "An Error Occured",
172                         ],
173                         color="danger",
174                     ),
175                 ]
176             )
177
178     def _add_navbar(self):
179         """Add nav element with navigation panel. It is placed on the top.
180         """
181         return dbc.NavbarSimple(
182             id="navbarsimple-main",
183             children=[
184                 dbc.NavItem(
185                     dbc.NavLink(
186                         "Continuous Performance Trending",
187                         external_link=True,
188                         href="#"
189                     )
190                 )
191             ],
192             brand="Dashboard",
193             brand_href="/",
194             brand_external_link=True,
195             class_name="p-2",
196             fluid=True,
197         )
198
199     def _add_ctrl_col(self) -> dbc.Col:
200         """Add column with controls. It is placed on the left side.
201         """
202         return dbc.Col(
203             id="col-controls",
204             children=[
205                 self._add_ctrl_panel(),
206                 self._add_ctrl_shown()
207             ],
208         )
209
210     def _add_plotting_col(self) -> dbc.Col:
211         """Add column with plots and tables. It is placed on the right side.
212         """
213         return dbc.Col(
214             id="col-plotting-area",
215             children=[
216                 dbc.Row(  # Throughput
217                     id="row-graph-tput",
218                     class_name="g-0 p-2",
219                     children=[
220                         dcc.Loading(
221                             dcc.Graph(id="graph-tput")
222                         )
223                     ]
224                 ),
225                 dbc.Row(  # Latency
226                     id="row-graph-lat",
227                     class_name="g-0 p-2",
228                     children=[
229                         dcc.Loading(
230                             dcc.Graph(id="graph-latency")
231                         )
232                     ]
233                 ),
234                 dbc.Row(  # Download
235                     id="div-download",
236                     class_name="g-0",
237                     children=[
238                         dcc.Loading(children=[
239                             dbc.Button(
240                                 id="btn-download-data",
241                                 children=["Download Data"]
242                             ),
243                             dcc.Download(id="download-data")
244                         ])
245                     ]
246                 )
247             ],
248             width=9,
249         )
250
251     def _add_ctrl_panel(self) -> dbc.Row:
252         """
253         """
254         return dbc.Row(
255             id="row-ctrl-panel",
256             class_name="g-0",
257             children=[
258                 dbc.Label("Physical Test Bed Topology, NIC and Driver"),
259                 dcc.Dropdown(
260                     id="dd-ctrl-phy",
261                     placeholder="Select a Physical Test Bed Topology...",
262                     multi=False,
263                     clearable=False,
264                     options=[
265                         {"label": k, "value": k} for k in self.spec_tbs.keys()
266                     ],
267                 ),
268                 dbc.Label("Area"),
269                 dcc.Dropdown(
270                     id="dd-ctrl-area",
271                     placeholder="Select an Area...",
272                     disabled=True,
273                     multi=False,
274                     clearable=False,
275                 ),
276                 dbc.Label("Test"),
277                 dcc.Dropdown(
278                     id="dd-ctrl-test",
279                     placeholder="Select a Test...",
280                     disabled=True,
281                     multi=False,
282                     clearable=False,
283                 ),
284                 dbc.Row(
285                     id="row-ctrl-core",
286                     class_name="g-0",
287                     children=[
288                         dbc.Label("Number of Cores"),
289                         dbc.Col([
290                             dbc.Checklist(
291                                 id="cl-ctrl-core-all",
292                                 options=[{"label": "All", "value": "all"}, ],
293                                 inline=False,
294                                 switch=False
295                             ),
296                         ], width=3),
297                         dbc.Col([
298                             dbc.Checklist(
299                                 id="cl-ctrl-core",
300                                 inline=True,
301                                 switch=False
302                             )
303                         ])
304                     ]
305                 ),
306                 dbc.Row(
307                     id="row-ctrl-framesize",
308                     class_name="g-0",
309                     children=[
310                         dbc.Label("Frame Size"),
311                         dbc.Col([
312                             dbc.Checklist(
313                                 id="cl-ctrl-framesize-all",
314                                 options=[{"label": "All", "value": "all"}, ],
315                                 inline=True,
316                                 switch=False
317                             ),
318                         ], width=3),
319                         dbc.Col([
320                             dbc.Checklist(
321                                 id="cl-ctrl-framesize",
322                                 inline=True,
323                                 switch=False
324                             )
325                         ])
326                     ]
327                 ),
328                 dbc.Row(
329                     id="row-ctrl-testtype",
330                     class_name="g-0",
331                     children=[
332                         dbc.Label("Test Type"),
333                         dbc.Col([
334                             dbc.Checklist(
335                                 id="cl-ctrl-testtype-all",
336                                 options=[{"label": "All", "value": "all"}, ],
337                                 inline=True,
338                                 switch=False
339                             ),
340                         ], width=3),
341                         dbc.Col([
342                             dbc.Checklist(
343                                 id="cl-ctrl-testtype",
344                                 inline=True,
345                                 switch=False
346                             )
347                         ])
348                     ]
349                 ),
350                 dbc.Row(
351                     class_name="g-0",
352                     children=[
353                         dbc.Button(
354                             id="btn-ctrl-add",
355                             children="Add",
356                         )
357                     ]
358                 ),
359                 dbc.Row(
360                     class_name="g-0",
361                     children=[
362                         dcc.DatePickerRange(
363                             id="dpr-period",
364                             min_date_allowed=\
365                                 datetime.utcnow()-timedelta(days=180),
366                             max_date_allowed=datetime.utcnow(),
367                             initial_visible_month=datetime.utcnow(),
368                             start_date=datetime.utcnow() - timedelta(days=180),
369                             end_date=datetime.utcnow(),
370                             display_format="D MMMM YY"
371                         )
372                     ]
373                 )
374             ]
375         )
376
377     def _add_ctrl_shown(self) -> dbc.Row:
378         """
379         """
380         return dbc.Row(
381             id="div-ctrl-shown",
382             class_name="g-0",
383             children=[
384                 dbc.Row(
385                     class_name="g-0",
386                     children=[
387                         dbc.Label("Selected tests"),
388                         dbc.Checklist(
389                             id="cl-selected",
390                             options=[],
391                             inline=False
392                         )
393                     ]
394                 ),
395                 dbc.Row(
396                     class_name="g-0",
397                     children=[
398                         dbc.ButtonGroup(
399                             [
400                                 dbc.Button(
401                                     id="btn-sel-remove",
402                                     children="Remove Selected",
403                                     color="secondary",
404                                     disabled=False
405                                 ),
406                                 dbc.Button(
407                                     id="btn-sel-remove-all",
408                                     children="Remove All",
409                                     color="secondary",
410                                     disabled=False
411                                 ),
412                                 dbc.Button(
413                                     id="btn-sel-display",
414                                     children="Display",
415                                     color="secondary",
416                                     disabled=False
417                                 )
418                             ],
419                             size="md",
420                             class_name="me-1",
421                         ),
422                     ]
423                 )
424             ]
425         )
426
427     def callbacks(self, app):
428
429         @app.callback(
430             Output("dd-ctrl-area", "options"),
431             Output("dd-ctrl-area", "disabled"),
432             Input("dd-ctrl-phy", "value"),
433         )
434         def _update_dd_area(phy):
435             """
436             """
437
438             if phy is None:
439                 raise PreventUpdate
440
441             try:
442                 options = [
443                     {"label": self.spec_tbs[phy][v]["label"], "value": v}
444                         for v in [v for v in self.spec_tbs[phy].keys()]
445                 ]
446                 disable = False
447             except KeyError:
448                 options = list()
449                 disable = True
450
451             return options, disable
452
453         @app.callback(
454             Output("dd-ctrl-test", "options"),
455             Output("dd-ctrl-test", "disabled"),
456             State("dd-ctrl-phy", "value"),
457             Input("dd-ctrl-area", "value"),
458         )
459         def _update_dd_test(phy, area):
460             """
461             """
462
463             if not area:
464                 raise PreventUpdate
465
466             try:
467                 options = [
468                     {"label": v, "value": v}
469                         for v in self.spec_tbs[phy][area]["test"]
470                 ]
471                 disable = False
472             except KeyError:
473                 options = list()
474                 disable = True
475
476             return options, disable
477
478         @app.callback(
479             Output("cl-ctrl-core", "options"),
480             Output("cl-ctrl-framesize", "options"),
481             Output("cl-ctrl-testtype", "options"),
482             State("dd-ctrl-phy", "value"),
483             State("dd-ctrl-area", "value"),
484             Input("dd-ctrl-test", "value"),
485         )
486         def _update_btn_add(phy, area, test):
487             """
488             """
489
490             if test is None:
491                 raise PreventUpdate
492
493             core_opts = []
494             framesize_opts = []
495             testtype_opts = []
496             if phy and area and test:
497                 core_opts = [
498                     {"label": v, "value": v}
499                         for v in self.spec_tbs[phy][area]["core"]
500                 ]
501                 framesize_opts = [
502                     {"label": v, "value": v}
503                         for v in self.spec_tbs[phy][area]["frame-size"]
504                 ]
505                 testtype_opts = [
506                     {"label": v, "value": v}
507                         for v in self.spec_tbs[phy][area]["test-type"]
508                 ]
509
510             return (
511                 core_opts,
512                 framesize_opts,
513                 testtype_opts,
514             )
515
516         def _sync_checklists(opt, sel, all, id):
517             """
518             """
519             options = {v["value"] for v in opt}
520             input_id = callback_context.triggered[0]["prop_id"].split(".")[0]
521             if input_id == id:
522                 all = ["all"] if set(sel) == options else list()
523             else:
524                 sel = list(options) if all else list()
525             return sel, all
526
527         @app.callback(
528             Output("cl-ctrl-core", "value"),
529             Output("cl-ctrl-core-all", "value"),
530             State("cl-ctrl-core", "options"),
531             Input("cl-ctrl-core", "value"),
532             Input("cl-ctrl-core-all", "value"),
533             prevent_initial_call=True
534         )
535         def _sync_cl_core(opt, sel, all):
536             return _sync_checklists(opt, sel, all, "cl-ctrl-core")
537
538         @app.callback(
539             Output("cl-ctrl-framesize", "value"),
540             Output("cl-ctrl-framesize-all", "value"),
541             State("cl-ctrl-framesize", "options"),
542             Input("cl-ctrl-framesize", "value"),
543             Input("cl-ctrl-framesize-all", "value"),
544             prevent_initial_call=True
545         )
546         def _sync_cl_framesize(opt, sel, all):
547             return _sync_checklists(opt, sel, all, "cl-ctrl-framesize")
548
549         @app.callback(
550             Output("cl-ctrl-testtype", "value"),
551             Output("cl-ctrl-testtype-all", "value"),
552             State("cl-ctrl-testtype", "options"),
553             Input("cl-ctrl-testtype", "value"),
554             Input("cl-ctrl-testtype-all", "value"),
555             prevent_initial_call=True
556         )
557         def _sync_cl_testtype(opt, sel, all):
558             return _sync_checklists(opt, sel, all, "cl-ctrl-testtype")
559
560         @app.callback(
561             Output("graph-tput", "figure"),
562             Output("graph-latency", "figure"),
563             Output("selected-tests", "data"),  # Store
564             Output("cl-selected", "options"),  # User selection
565             Output("dd-ctrl-phy", "value"),
566             Output("dd-ctrl-area", "value"),
567             Output("dd-ctrl-test", "value"),
568             State("selected-tests", "data"),  # Store
569             State("cl-selected", "value"),
570             State("dd-ctrl-phy", "value"),
571             State("dd-ctrl-area", "value"),
572             State("dd-ctrl-test", "value"),
573             State("cl-ctrl-core", "value"),
574             State("cl-ctrl-framesize", "value"),
575             State("cl-ctrl-testtype", "value"),
576             Input("btn-ctrl-add", "n_clicks"),
577             Input("btn-sel-display", "n_clicks"),
578             Input("btn-sel-remove", "n_clicks"),
579             Input("btn-sel-remove-all", "n_clicks"),
580             Input("dpr-period", "start_date"),
581             Input("dpr-period", "end_date"),
582             prevent_initial_call=True
583         )
584         def _process_list(store_sel, list_sel, phy, area, test, cores,
585                 framesizes, testtypes, btn_add, btn_display, btn_remove,
586                 btn_remove_all, d_start, d_end):
587             """
588             """
589
590             if not (btn_add or btn_display or btn_remove or btn_remove_all or \
591                     d_start or d_end):
592                 raise PreventUpdate
593
594             def _list_tests():
595                 # Display selected tests with checkboxes:
596                 if store_sel:
597                     return [
598                         {"label": v["id"], "value": v["id"]} for v in store_sel
599                     ]
600                 else:
601                     return list()
602
603             class RetunValue:
604                 def __init__(self) -> None:
605                     self._output = {
606                         "graph-tput-figure": no_update,
607                         "graph-lat-figure": no_update,
608                         "selected-tests-data": no_update,
609                         "cl-selected-options": no_update,
610                         "dd-ctrl-phy-value": no_update,
611                         "dd-ctrl-area-value": no_update,
612                         "dd-ctrl-test-value": no_update,
613                     }
614
615                 def value(self):
616                     return tuple(self._output.values())
617
618                 def set_values(self, kwargs: dict) -> None:
619                     for key, val in kwargs.items():
620                         if key in self._output:
621                             self._output[key] = val
622                         else:
623                             raise KeyError(f"The key {key} is not defined.")
624
625
626             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
627
628             d_start = datetime(int(d_start[0:4]), int(d_start[5:7]),
629                 int(d_start[8:10]))
630             d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10]))
631
632             output = RetunValue()
633
634             if trigger_id == "btn-ctrl-add":
635                 # Add selected test to the list of tests in store:
636                 if phy and area and test and cores and framesizes and testtypes:
637                     if store_sel is None:
638                         store_sel = list()
639                     for core in cores:
640                         for framesize in framesizes:
641                             for ttype in testtypes:
642                                 tid = (
643                                     f"{phy.replace('af_xdp', 'af-xdp')}-"
644                                     f"{area}-"
645                                     f"{framesize.lower()}-"
646                                     f"{core.lower()}-"
647                                     f"{test}-"
648                                     f"{ttype.lower()}"
649                                 )
650                                 if tid not in [itm["id"] for itm in store_sel]:
651                                     store_sel.append({
652                                         "id": tid,
653                                         "phy": phy,
654                                         "area": area,
655                                         "test": test,
656                                         "framesize": framesize.lower(),
657                                         "core": core.lower(),
658                                         "testtype": ttype.lower()
659                                     })
660                 output.set_values({
661                     "selected-tests-data": store_sel,
662                     "cl-selected-options": _list_tests(),
663                     "dd-ctrl-phy-value": None,
664                     "dd-ctrl-area-value": None,
665                     "dd-ctrl-test-value": None,
666                 })
667
668             elif trigger_id in ("btn-sel-display", "dpr-period"):
669                 fig_tput, fig_lat = graph_trending(
670                     self.data, store_sel, self.layout, d_start, d_end
671                 )
672                 output.set_values({
673                     "graph-tput-figure": \
674                         fig_tput if fig_tput else self.NO_GRAPH,
675                     "graph-lat-figure": \
676                         fig_lat if fig_lat else self.NO_GRAPH,
677                 })
678             elif trigger_id == "btn-sel-remove-all":
679                 output.set_values({
680                     "graph-tput-figure": self.NO_GRAPH,
681                     "graph-lat-figure": self.NO_GRAPH,
682                     "selected-tests-data": list(),
683                     "cl-selected-options": list()
684                 })
685             elif trigger_id == "btn-sel-remove":
686                 if list_sel:
687                     new_store_sel = list()
688                     for item in store_sel:
689                         if item["id"] not in list_sel:
690                             new_store_sel.append(item)
691                     store_sel = new_store_sel
692                 if store_sel:
693                     fig_tput, fig_lat = graph_trending(
694                         self.data, store_sel, self.layout, d_start, d_end
695                     )
696                     output.set_values({
697                         "graph-tput-figure": \
698                             fig_tput if fig_tput else self.NO_GRAPH,
699                         "graph-lat-figure": \
700                             fig_lat if fig_lat else self.NO_GRAPH,
701                         "selected-tests-data": store_sel,
702                         "cl-selected-options": _list_tests()
703                     })
704                 else:
705                     output.set_values({
706                         "graph-tput-figure": self.NO_GRAPH,
707                         "graph-lat-figure": self.NO_GRAPH,
708                         "selected-tests-data": store_sel,
709                         "cl-selected-options": _list_tests()
710                     })
711
712             return output.value()
713
714         @app.callback(
715             Output("metadata-tput-lat", "children"),
716             Output("metadata-hdrh-graph", "children"),
717             Output("offcanvas-metadata", "is_open"),
718             Input("graph-tput", "clickData"),
719             Input("graph-latency", "clickData")
720         )
721         def _show_tput_metadata(tput_data, lat_data) -> dbc.Card:
722             """
723             """
724             if not (tput_data or lat_data):
725                 raise PreventUpdate
726
727             metadata = no_update
728             graph = list()
729
730             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
731             if trigger_id == "graph-tput":
732                 title = "Throughput"
733                 txt = tput_data["points"][0]["text"].replace("<br>", "\n")
734             elif trigger_id == "graph-latency":
735                 title = "Latency"
736                 txt = lat_data["points"][0]["text"].replace("<br>", "\n")
737                 hdrh_data = lat_data["points"][0].get("customdata", None)
738                 if hdrh_data:
739                     graph = [dcc.Graph(
740                         id="hdrh-latency-graph",
741                         figure=graph_hdrh_latency(hdrh_data, self.layout)
742                     ), ]
743
744             metadata = [
745                 dbc.Card(
746                     children=[
747                         dbc.CardHeader(title),
748                         dbc.CardBody(children=[txt])
749                     ]
750                 )
751             ]
752
753             return metadata, graph, True
754
755         @app.callback(
756             Output("download-data", "data"),
757             State("selected-tests", "data"),
758             Input("btn-download-data", "n_clicks"),
759             prevent_initial_call=True
760         )
761         def _download_data(store_sel, n_clicks):
762             """
763             """
764
765             if not n_clicks:
766                 raise PreventUpdate
767
768             df = pd.DataFrame()
769             for itm in store_sel:
770                 sel_data = select_trending_data(self.data, itm)
771                 if sel_data is None:
772                     continue
773                 df = pd.concat([df, sel_data], ignore_index=True)
774
775             return dcc.send_data_frame(df.to_csv, "trending_data.csv")