feat(uti): Cover theme sync
[csit.git] / resources / tools / dash / app / pal / trending / layout.py
1 # Copyright (c) 2022 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Plotly Dash HTML layout override.
15 """
16
17 import pandas as pd
18
19 from dash import dcc
20 from dash import html
21 from dash import callback_context, no_update
22 from dash import Input, Output, State
23 from dash.exceptions import PreventUpdate
24 import dash_bootstrap_components as dbc
25 from yaml import load, FullLoader, YAMLError
26 from datetime import datetime, timedelta
27
28 from ..data.data import Data
29 from .graphs import graph_trending, graph_hdrh_latency, \
30     select_trending_data
31
32
33 class Layout:
34     """
35     """
36
37     NO_GRAPH = {"data": [], "layout": {}, "frames": []}
38
39     def __init__(self, app, html_layout_file, spec_file, graph_layout_file,
40         data_spec_file):
41         """
42         """
43
44         # Inputs
45         self._app = app
46         self._html_layout_file = html_layout_file
47         self._spec_file = spec_file
48         self._graph_layout_file = graph_layout_file
49         self._data_spec_file = data_spec_file
50
51         # Read the data:
52         data_mrr = Data(
53             data_spec_file=self._data_spec_file,
54             debug=True
55         ).read_trending_mrr(days=5)
56
57         data_ndrpdr = Data(
58             data_spec_file=self._data_spec_file,
59             debug=True
60         ).read_trending_ndrpdr(days=14)
61
62         self._data = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
63
64         # Read from files:
65         self._html_layout = ""
66         self._spec_tbs = None
67         self._graph_layout = None
68
69         try:
70             with open(self._html_layout_file, "r") as file_read:
71                 self._html_layout = file_read.read()
72         except IOError as err:
73             raise RuntimeError(
74                 f"Not possible to open the file {self._html_layout_file}\n{err}"
75             )
76
77         try:
78             with open(self._spec_file, "r") as file_read:
79                 self._spec_tbs = load(file_read, Loader=FullLoader)
80         except IOError as err:
81             raise RuntimeError(
82                 f"Not possible to open the file {self._spec_file,}\n{err}"
83             )
84         except YAMLError as err:
85             raise RuntimeError(
86                 f"An error occurred while parsing the specification file "
87                 f"{self._spec_file,}\n"
88                 f"{err}"
89             )
90
91         try:
92             with open(self._graph_layout_file, "r") as file_read:
93                 self._graph_layout = load(file_read, Loader=FullLoader)
94         except IOError as err:
95             raise RuntimeError(
96                 f"Not possible to open the file {self._graph_layout_file}\n"
97                 f"{err}"
98             )
99         except YAMLError as err:
100             raise RuntimeError(
101                 f"An error occurred while parsing the specification file "
102                 f"{self._graph_layout_file}\n"
103                 f"{err}"
104             )
105
106         # Callbacks:
107         if self._app is not None and hasattr(self, 'callbacks'):
108             self.callbacks(self._app)
109
110     @property
111     def html_layout(self):
112         return self._html_layout
113
114     @property
115     def spec_tbs(self):
116         return self._spec_tbs
117
118     @property
119     def data(self):
120         return self._data
121
122     @property
123     def layout(self):
124         return self._graph_layout
125
126     def add_content(self):
127         """
128         """
129         if self.html_layout and self.spec_tbs:
130             return html.Div(
131                 id="div-main",
132                 children=[
133                     dbc.Row(
134                         id="row-navbar",
135                         class_name="g-0",
136                         children=[
137                             self._add_navbar(),
138                         ]
139                     ),
140                     dcc.Loading(
141                         dbc.Offcanvas(
142                             id="offcanvas-metadata",
143                             title="Throughput And Latency",
144                             placement="end",
145                             is_open=False,
146                             children=[
147                                 dbc.Row(id="metadata-tput-lat"),
148                                 dbc.Row(id="metadata-hdrh-graph"),
149                             ]
150                         )
151                     ),
152                     dbc.Row(
153                         id="row-main",
154                         class_name="g-0 p-2",
155                         children=[
156                             dcc.Store(
157                                 id="selected-tests"
158                             ),
159                             self._add_ctrl_col(),
160                             self._add_plotting_col(),
161                         ]
162                     )
163                 ]
164             )
165         else:
166             return html.Div(
167                 id="div-main-error",
168                 children=[
169                     dbc.Alert(
170                         [
171                             "An Error Occured",
172                         ],
173                         color="danger",
174                     ),
175                 ]
176             )
177
178     def _add_navbar(self):
179         """Add nav element with navigation panel. It is placed on the top.
180         """
181         return dbc.NavbarSimple(
182             id="navbarsimple-main",
183             children=[
184                 dbc.NavItem(
185                     dbc.NavLink(
186                         "Continuous Performance Trending",
187                         disabled=True,
188                         external_link=True,
189                         href="#"
190                     )
191                 )
192             ],
193             brand="Dashboard",
194             brand_href="/",
195             brand_external_link=True,
196             class_name="p-2",
197             fluid=True,
198         )
199
200     def _add_ctrl_col(self) -> dbc.Col:
201         """Add column with controls. It is placed on the left side.
202         """
203         return dbc.Col(
204             id="col-controls",
205             children=[
206                 self._add_ctrl_panel(),
207                 self._add_ctrl_shown()
208             ],
209         )
210
211     def _add_plotting_col(self) -> dbc.Col:
212         """Add column with plots and tables. It is placed on the right side.
213         """
214         return dbc.Col(
215             id="col-plotting-area",
216             children=[
217                 dbc.Row(  # Throughput
218                     id="row-graph-tput",
219                     class_name="g-0 p-2",
220                     children=[
221                         dcc.Loading(
222                             dcc.Graph(id="graph-tput")
223                         )
224                     ]
225                 ),
226                 dbc.Row(  # Latency
227                     id="row-graph-lat",
228                     class_name="g-0 p-2",
229                     children=[
230                         dcc.Loading(
231                             dcc.Graph(id="graph-latency")
232                         )
233                     ]
234                 ),
235                 dbc.Row(  # Download
236                     id="div-download",
237                     class_name="g-0",
238                     children=[
239                         dcc.Loading(children=[
240                             dbc.Button(
241                                 id="btn-download-data",
242                                 children=["Download Data"]
243                             ),
244                             dcc.Download(id="download-data")
245                         ])
246                     ]
247                 )
248             ],
249             width=9,
250         )
251
252     def _add_ctrl_panel(self) -> dbc.Row:
253         """
254         """
255         return dbc.Row(
256             id="row-ctrl-panel",
257             class_name="g-0",
258             children=[
259                 dbc.Label("Physical Test Bed Topology, NIC and Driver"),
260                 dbc.Select(
261                     id="dd-ctrl-phy",
262                     className="p-2",
263                     placeholder="Select a Physical Test Bed Topology...",
264                     options=[
265                         {"label": k, "value": k} for k in self.spec_tbs.keys()
266                     ],
267                     size="sm",
268                 ),
269                 dbc.Label("Area"),
270                 dbc.Select(
271                     id="dd-ctrl-area",
272                     className="p-2",
273                     placeholder="Select an Area...",
274                     disabled=True,
275                     size="sm",
276                 ),
277                 dbc.Label("Test"),
278                 dbc.Select(
279                     id="dd-ctrl-test",
280                     className="p-2",
281                     placeholder="Select a Test...",
282                     disabled=True,
283                     size="sm",
284                 ),
285                 dbc.Row(
286                     id="row-ctrl-core",
287                     class_name="g-0",
288                     children=[
289                         dbc.Label("Number of Cores"),
290                         dbc.Col([
291                             dbc.Checklist(
292                                 id="cl-ctrl-core-all",
293                                 options=[{"label": "All", "value": "all"}, ],
294                                 inline=False,
295                                 switch=False
296                             ),
297                         ], width=3),
298                         dbc.Col([
299                             dbc.Checklist(
300                                 id="cl-ctrl-core",
301                                 inline=True,
302                                 switch=False
303                             )
304                         ])
305                     ]
306                 ),
307                 dbc.Row(
308                     id="row-ctrl-framesize",
309                     class_name="g-0",
310                     children=[
311                         dbc.Label("Frame Size"),
312                         dbc.Col([
313                             dbc.Checklist(
314                                 id="cl-ctrl-framesize-all",
315                                 options=[{"label": "All", "value": "all"}, ],
316                                 inline=True,
317                                 switch=False
318                             ),
319                         ], width=3),
320                         dbc.Col([
321                             dbc.Checklist(
322                                 id="cl-ctrl-framesize",
323                                 inline=True,
324                                 switch=False
325                             )
326                         ])
327                     ]
328                 ),
329                 dbc.Row(
330                     id="row-ctrl-testtype",
331                     class_name="g-0",
332                     children=[
333                         dbc.Label("Test Type"),
334                         dbc.Col([
335                             dbc.Checklist(
336                                 id="cl-ctrl-testtype-all",
337                                 options=[{"label": "All", "value": "all"}, ],
338                                 inline=True,
339                                 switch=False
340                             ),
341                         ], width=3),
342                         dbc.Col([
343                             dbc.Checklist(
344                                 id="cl-ctrl-testtype",
345                                 inline=True,
346                                 switch=False
347                             )
348                         ])
349                     ]
350                 ),
351                 dbc.Row(
352                     class_name="g-0",
353                     children=[
354                         dbc.Button(
355                             id="btn-ctrl-add",
356                             children="Add",
357                         )
358                     ]
359                 ),
360                 dbc.Row(
361                     class_name="g-0",
362                     children=[
363                         dcc.DatePickerRange(
364                             id="dpr-period",
365                             min_date_allowed=\
366                                 datetime.utcnow()-timedelta(days=180),
367                             max_date_allowed=datetime.utcnow(),
368                             initial_visible_month=datetime.utcnow(),
369                             start_date=datetime.utcnow() - timedelta(days=180),
370                             end_date=datetime.utcnow(),
371                             display_format="D MMMM YY"
372                         )
373                     ]
374                 )
375             ]
376         )
377
378     def _add_ctrl_shown(self) -> dbc.Row:
379         """
380         """
381         return dbc.Row(
382             id="div-ctrl-shown",
383             class_name="g-0",
384             children=[
385                 dbc.Row(
386                     class_name="g-0",
387                     children=[
388                         dbc.Label("Selected tests"),
389                         dbc.Checklist(
390                             id="cl-selected",
391                             options=[],
392                             inline=False
393                         )
394                     ]
395                 ),
396                 dbc.Row(
397                     class_name="g-0",
398                     children=[
399                         dbc.ButtonGroup(
400                             [
401                                 dbc.Button(
402                                     id="btn-sel-remove-all",
403                                     children="Remove All",
404                                     color="secondary",
405                                     disabled=False
406                                 ),
407                                 dbc.Button(
408                                     id="btn-sel-remove",
409                                     children="Remove Selected",
410                                     color="secondary",
411                                     disabled=False
412                                 ),
413                                 dbc.Button(
414                                     id="btn-sel-display",
415                                     children="Display",
416                                     color="secondary",
417                                     disabled=False
418                                 )
419                             ],
420                             size="md",
421                             class_name="me-1",
422                         ),
423                     ]
424                 )
425             ]
426         )
427
428     def callbacks(self, app):
429
430         @app.callback(
431             Output("dd-ctrl-area", "options"),
432             Output("dd-ctrl-area", "disabled"),
433             Input("dd-ctrl-phy", "value"),
434         )
435         def _update_dd_area(phy):
436             """
437             """
438
439             if phy is None:
440                 raise PreventUpdate
441
442             try:
443                 options = [
444                     {"label": self.spec_tbs[phy][v]["label"], "value": v}
445                         for v in [v for v in self.spec_tbs[phy].keys()]
446                 ]
447                 disable = False
448             except KeyError:
449                 options = list()
450                 disable = True
451
452             return options, disable
453
454         @app.callback(
455             Output("dd-ctrl-test", "options"),
456             Output("dd-ctrl-test", "disabled"),
457             State("dd-ctrl-phy", "value"),
458             Input("dd-ctrl-area", "value"),
459         )
460         def _update_dd_test(phy, area):
461             """
462             """
463
464             if not area:
465                 raise PreventUpdate
466
467             try:
468                 options = [
469                     {"label": v, "value": v}
470                         for v in self.spec_tbs[phy][area]["test"]
471                 ]
472                 disable = False
473             except KeyError:
474                 options = list()
475                 disable = True
476
477             return options, disable
478
479         @app.callback(
480             Output("cl-ctrl-core", "options"),
481             Output("cl-ctrl-framesize", "options"),
482             Output("cl-ctrl-testtype", "options"),
483             State("dd-ctrl-phy", "value"),
484             State("dd-ctrl-area", "value"),
485             Input("dd-ctrl-test", "value"),
486         )
487         def _update_btn_add(phy, area, test):
488             """
489             """
490
491             if test is None:
492                 raise PreventUpdate
493
494             core_opts = []
495             framesize_opts = []
496             testtype_opts = []
497             if phy and area and test:
498                 core_opts = [
499                     {"label": v, "value": v}
500                         for v in self.spec_tbs[phy][area]["core"]
501                 ]
502                 framesize_opts = [
503                     {"label": v, "value": v}
504                         for v in self.spec_tbs[phy][area]["frame-size"]
505                 ]
506                 testtype_opts = [
507                     {"label": v, "value": v}
508                         for v in self.spec_tbs[phy][area]["test-type"]
509                 ]
510
511             return (
512                 core_opts,
513                 framesize_opts,
514                 testtype_opts,
515             )
516
517         def _sync_checklists(opt, sel, all, id):
518             """
519             """
520             options = {v["value"] for v in opt}
521             input_id = callback_context.triggered[0]["prop_id"].split(".")[0]
522             if input_id == id:
523                 all = ["all"] if set(sel) == options else list()
524             else:
525                 sel = list(options) if all else list()
526             return sel, all
527
528         @app.callback(
529             Output("cl-ctrl-core", "value"),
530             Output("cl-ctrl-core-all", "value"),
531             State("cl-ctrl-core", "options"),
532             Input("cl-ctrl-core", "value"),
533             Input("cl-ctrl-core-all", "value"),
534             prevent_initial_call=True
535         )
536         def _sync_cl_core(opt, sel, all):
537             return _sync_checklists(opt, sel, all, "cl-ctrl-core")
538
539         @app.callback(
540             Output("cl-ctrl-framesize", "value"),
541             Output("cl-ctrl-framesize-all", "value"),
542             State("cl-ctrl-framesize", "options"),
543             Input("cl-ctrl-framesize", "value"),
544             Input("cl-ctrl-framesize-all", "value"),
545             prevent_initial_call=True
546         )
547         def _sync_cl_framesize(opt, sel, all):
548             return _sync_checklists(opt, sel, all, "cl-ctrl-framesize")
549
550         @app.callback(
551             Output("cl-ctrl-testtype", "value"),
552             Output("cl-ctrl-testtype-all", "value"),
553             State("cl-ctrl-testtype", "options"),
554             Input("cl-ctrl-testtype", "value"),
555             Input("cl-ctrl-testtype-all", "value"),
556             prevent_initial_call=True
557         )
558         def _sync_cl_testtype(opt, sel, all):
559             return _sync_checklists(opt, sel, all, "cl-ctrl-testtype")
560
561         @app.callback(
562             Output("graph-tput", "figure"),
563             Output("graph-latency", "figure"),
564             Output("selected-tests", "data"),  # Store
565             Output("cl-selected", "options"),  # User selection
566             Output("dd-ctrl-phy", "value"),
567             Output("dd-ctrl-area", "value"),
568             Output("dd-ctrl-test", "value"),
569             State("selected-tests", "data"),  # Store
570             State("cl-selected", "value"),
571             State("dd-ctrl-phy", "value"),
572             State("dd-ctrl-area", "value"),
573             State("dd-ctrl-test", "value"),
574             State("cl-ctrl-core", "value"),
575             State("cl-ctrl-framesize", "value"),
576             State("cl-ctrl-testtype", "value"),
577             Input("btn-ctrl-add", "n_clicks"),
578             Input("btn-sel-display", "n_clicks"),
579             Input("btn-sel-remove", "n_clicks"),
580             Input("btn-sel-remove-all", "n_clicks"),
581             Input("dpr-period", "start_date"),
582             Input("dpr-period", "end_date"),
583             prevent_initial_call=True
584         )
585         def _process_list(store_sel, list_sel, phy, area, test, cores,
586                 framesizes, testtypes, btn_add, btn_display, btn_remove,
587                 btn_remove_all, d_start, d_end):
588             """
589             """
590
591             if not (btn_add or btn_display or btn_remove or btn_remove_all or \
592                     d_start or d_end):
593                 raise PreventUpdate
594
595             def _list_tests():
596                 # Display selected tests with checkboxes:
597                 if store_sel:
598                     return [
599                         {"label": v["id"], "value": v["id"]} for v in store_sel
600                     ]
601                 else:
602                     return list()
603
604             class RetunValue:
605                 def __init__(self) -> None:
606                     self._output = {
607                         "graph-tput-figure": no_update,
608                         "graph-lat-figure": no_update,
609                         "selected-tests-data": no_update,
610                         "cl-selected-options": no_update,
611                         "dd-ctrl-phy-value": no_update,
612                         "dd-ctrl-area-value": no_update,
613                         "dd-ctrl-test-value": no_update,
614                     }
615
616                 def value(self):
617                     return tuple(self._output.values())
618
619                 def set_values(self, kwargs: dict) -> None:
620                     for key, val in kwargs.items():
621                         if key in self._output:
622                             self._output[key] = val
623                         else:
624                             raise KeyError(f"The key {key} is not defined.")
625
626
627             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
628
629             d_start = datetime(int(d_start[0:4]), int(d_start[5:7]),
630                 int(d_start[8:10]))
631             d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10]))
632
633             output = RetunValue()
634
635             if trigger_id == "btn-ctrl-add":
636                 # Add selected test to the list of tests in store:
637                 if phy and area and test and cores and framesizes and testtypes:
638                     if store_sel is None:
639                         store_sel = list()
640                     for core in cores:
641                         for framesize in framesizes:
642                             for ttype in testtypes:
643                                 tid = (
644                                     f"{phy.replace('af_xdp', 'af-xdp')}-"
645                                     f"{area}-"
646                                     f"{framesize.lower()}-"
647                                     f"{core.lower()}-"
648                                     f"{test}-"
649                                     f"{ttype.lower()}"
650                                 )
651                                 if tid not in [itm["id"] for itm in store_sel]:
652                                     store_sel.append({
653                                         "id": tid,
654                                         "phy": phy,
655                                         "area": area,
656                                         "test": test,
657                                         "framesize": framesize.lower(),
658                                         "core": core.lower(),
659                                         "testtype": ttype.lower()
660                                     })
661                 output.set_values({
662                     "selected-tests-data": store_sel,
663                     "cl-selected-options": _list_tests(),
664                     "dd-ctrl-phy-value": None,
665                     "dd-ctrl-area-value": None,
666                     "dd-ctrl-test-value": None,
667                 })
668
669             elif trigger_id in ("btn-sel-display", "dpr-period"):
670                 fig_tput, fig_lat = graph_trending(
671                     self.data, store_sel, self.layout, d_start, d_end
672                 )
673                 output.set_values({
674                     "graph-tput-figure": \
675                         fig_tput if fig_tput else self.NO_GRAPH,
676                     "graph-lat-figure": \
677                         fig_lat if fig_lat else self.NO_GRAPH,
678                 })
679             elif trigger_id == "btn-sel-remove-all":
680                 output.set_values({
681                     "graph-tput-figure": self.NO_GRAPH,
682                     "graph-lat-figure": self.NO_GRAPH,
683                     "selected-tests-data": list(),
684                     "cl-selected-options": list()
685                 })
686             elif trigger_id == "btn-sel-remove":
687                 if list_sel:
688                     new_store_sel = list()
689                     for item in store_sel:
690                         if item["id"] not in list_sel:
691                             new_store_sel.append(item)
692                     store_sel = new_store_sel
693                 if store_sel:
694                     fig_tput, fig_lat = graph_trending(
695                         self.data, store_sel, self.layout, d_start, d_end
696                     )
697                     output.set_values({
698                         "graph-tput-figure": \
699                             fig_tput if fig_tput else self.NO_GRAPH,
700                         "graph-lat-figure": \
701                             fig_lat if fig_lat else self.NO_GRAPH,
702                         "selected-tests-data": store_sel,
703                         "cl-selected-options": _list_tests()
704                     })
705                 else:
706                     output.set_values({
707                         "graph-tput-figure": self.NO_GRAPH,
708                         "graph-lat-figure": self.NO_GRAPH,
709                         "selected-tests-data": store_sel,
710                         "cl-selected-options": _list_tests()
711                     })
712
713             return output.value()
714
715         @app.callback(
716             Output("metadata-tput-lat", "children"),
717             Output("metadata-hdrh-graph", "children"),
718             Output("offcanvas-metadata", "is_open"),
719             Input("graph-tput", "clickData"),
720             Input("graph-latency", "clickData")
721         )
722         def _show_tput_metadata(tput_data, lat_data) -> dbc.Card:
723             """
724             """
725             if not (tput_data or lat_data):
726                 raise PreventUpdate
727
728             metadata = no_update
729             graph = list()
730
731             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
732             if trigger_id == "graph-tput":
733                 title = "Throughput"
734                 txt = tput_data["points"][0]["text"].replace("<br>", "\n")
735             elif trigger_id == "graph-latency":
736                 title = "Latency"
737                 txt = lat_data["points"][0]["text"].replace("<br>", "\n")
738                 hdrh_data = lat_data["points"][0].get("customdata", None)
739                 if hdrh_data:
740                     graph = [dcc.Graph(
741                         id="hdrh-latency-graph",
742                         figure=graph_hdrh_latency(hdrh_data, self.layout)
743                     ), ]
744
745             metadata = [
746                 dbc.Card(
747                     children=[
748                         dbc.CardHeader(children=[
749                             dcc.Clipboard(
750                                 target_id="tput-lat-metadata",
751                                 title="Copy",
752                                 style={"display": "inline-block"}
753                             ),
754                             title
755                         ]),
756                         dbc.CardBody(
757                             id="tput-lat-metadata",
758                             children=[txt]
759                         )
760                     ]
761                 )
762             ]
763
764             return metadata, graph, True
765
766         @app.callback(
767             Output("download-data", "data"),
768             State("selected-tests", "data"),
769             Input("btn-download-data", "n_clicks"),
770             prevent_initial_call=True
771         )
772         def _download_data(store_sel, n_clicks):
773             """
774             """
775
776             if not n_clicks:
777                 raise PreventUpdate
778
779             df = pd.DataFrame()
780             for itm in store_sel:
781                 sel_data = select_trending_data(self.data, itm)
782                 if sel_data is None:
783                     continue
784                 df = pd.concat([df, sel_data], ignore_index=True)
785
786             return dcc.send_data_frame(df.to_csv, "trending_data.csv")