feat(uti): Refactor grid layout
[csit.git] / resources / tools / dash / app / pal / trending / layout.py
1 # Copyright (c) 2022 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Plotly Dash HTML layout override.
15 """
16
17 import logging
18
19 import pandas as pd
20
21 from dash import dcc
22 from dash import html
23 from dash import callback_context, no_update
24 from dash import Input, Output, State
25 from dash.exceptions import PreventUpdate
26 import dash_bootstrap_components as dbc
27 from yaml import load, FullLoader, YAMLError
28 from datetime import datetime, timedelta
29
30 from ..data.data import Data
31 from .graphs import graph_trending, graph_hdrh_latency, \
32     select_trending_data
33
34
35 class Layout:
36     """
37     """
38
39     STYLE_HIDEN = {"display": "none"}
40     STYLE_BLOCK = {"display": "block", "vertical-align": "top"}
41     STYLE_INLINE ={
42         "display": "inline-block",
43         "vertical-align": "top"
44     }
45     NO_GRAPH = {"data": [], "layout": {}, "frames": []}
46
47     def __init__(self, app, html_layout_file, spec_file, graph_layout_file,
48         data_spec_file):
49         """
50         """
51
52         # Inputs
53         self._app = app
54         self._html_layout_file = html_layout_file
55         self._spec_file = spec_file
56         self._graph_layout_file = graph_layout_file
57         self._data_spec_file = data_spec_file
58
59         # Read the data:
60         data_mrr = Data(
61             data_spec_file=self._data_spec_file,
62             debug=True
63         ).read_trending_mrr(days=5)
64
65         data_ndrpdr = Data(
66             data_spec_file=self._data_spec_file,
67             debug=True
68         ).read_trending_ndrpdr(days=14)
69
70         self._data = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
71
72         # Read from files:
73         self._html_layout = ""
74         self._spec_tbs = None
75         self._graph_layout = None
76
77         try:
78             with open(self._html_layout_file, "r") as file_read:
79                 self._html_layout = file_read.read()
80         except IOError as err:
81             raise RuntimeError(
82                 f"Not possible to open the file {self._html_layout_file}\n{err}"
83             )
84
85         try:
86             with open(self._spec_file, "r") as file_read:
87                 self._spec_tbs = load(file_read, Loader=FullLoader)
88         except IOError as err:
89             raise RuntimeError(
90                 f"Not possible to open the file {self._spec_file,}\n{err}"
91             )
92         except YAMLError as err:
93             raise RuntimeError(
94                 f"An error occurred while parsing the specification file "
95                 f"{self._spec_file,}\n"
96                 f"{err}"
97             )
98
99         try:
100             with open(self._graph_layout_file, "r") as file_read:
101                 self._graph_layout = load(file_read, Loader=FullLoader)
102         except IOError as err:
103             raise RuntimeError(
104                 f"Not possible to open the file {self._graph_layout_file}\n"
105                 f"{err}"
106             )
107         except YAMLError as err:
108             raise RuntimeError(
109                 f"An error occurred while parsing the specification file "
110                 f"{self._graph_layout_file}\n"
111                 f"{err}"
112             )
113
114         # Callbacks:
115         if self._app is not None and hasattr(self, 'callbacks'):
116             self.callbacks(self._app)
117
118     @property
119     def html_layout(self):
120         return self._html_layout
121
122     @property
123     def spec_tbs(self):
124         return self._spec_tbs
125
126     @property
127     def data(self):
128         return self._data
129
130     @property
131     def layout(self):
132         return self._graph_layout
133
134     def add_content(self):
135         """
136         """
137         if self.html_layout and self.spec_tbs:
138             return html.Div(
139                 id="div-main",
140                 children=[
141                     dbc.Row(
142                         id="row-navbar",
143                         className="g-0",
144                         children=[
145                             self._add_navbar(),
146                         ]
147                     ),
148                     dbc.Row(
149                         id="row-main",
150                         className="g-0",
151                         children=[
152                             dcc.Store(
153                                 id="selected-tests"
154                             ),
155                             self._add_ctrl_col(),
156                             self._add_plotting_col(),
157                         ]
158                     ),
159                     dbc.Offcanvas(
160                         id="offcanvas-metadata",
161                         title="Throughput And Latency",
162                         placement="end",
163                         is_open=True,
164                         children=[
165                             html.P(
166                                 id="metadata",
167                                 children=[
168                                     "This is the placeholder for metadata."
169                                 ],
170                             )
171                         ]
172                     )
173                 ]
174             )
175         else:
176             return html.Div(
177                 id="div-main-error",
178                 children=[
179                     dbc.Alert(
180                         [
181                             "An Error Occured",
182                         ],
183                         color="danger",
184                     ),
185                 ]
186             )
187
188     def _add_navbar(self):
189         """Add nav element with navigation panel. It is placed on the top.
190         """
191         return dbc.NavbarSimple(
192             id="navbarsimple-main",
193             children=[
194                 dbc.NavItem(
195                     dbc.NavLink(
196                         "Continuous Performance Trending",
197                         external_link=True,
198                         href="#"
199                     )
200                 )
201             ],
202             brand="Dashboard",
203             brand_href="/",
204             brand_external_link=True,
205             #color="dark",
206             #dark=True,
207             fluid=True,
208         )
209
210     def _add_ctrl_col(self) -> dbc.Col:
211         """Add column with controls. It is placed on the left side.
212         """
213         return dbc.Col(
214             id="col-controls",
215             children=[
216                 self._add_ctrl_panel(),
217                 self._add_ctrl_shown()
218             ],
219         )
220
221     def _add_plotting_col(self) -> dbc.Col:
222         """Add column with plots and tables. It is placed on the right side.
223         """
224         return dbc.Col(
225             id="col-plotting-area",
226             children=[
227                 # Empty for now
228             ],
229             width=9,
230         )
231
232     def _add_ctrl_panel(self) -> dbc.Row:
233         """
234         """
235         return dbc.Row(
236             id="row-ctrl-panel",
237             className="g-0",
238             children=[
239                 dbc.Label("Physical Test Bed Topology, NIC and Driver"),
240                 dcc.Dropdown(
241                     id="dd-ctrl-phy",
242                     placeholder="Select a Physical Test Bed Topology...",
243                     multi=False,
244                     clearable=False,
245                     options=[
246                         {"label": k, "value": k} for k in self.spec_tbs.keys()
247                     ],
248                 ),
249                 dbc.Label("Area"),
250                 dcc.Dropdown(
251                     id="dd-ctrl-area",
252                     placeholder="Select an Area...",
253                     disabled=True,
254                     multi=False,
255                     clearable=False,
256                 ),
257                 dbc.Label("Test"),
258                 dcc.Dropdown(
259                     id="dd-ctrl-test",
260                     placeholder="Select a Test...",
261                     disabled=True,
262                     multi=False,
263                     clearable=False,
264                 ),
265                 dbc.Row(
266                     id="row-ctrl-core",
267                     className="g-0",
268                     children=[
269                         dbc.Label("Number of Cores"),
270                         dbc.Checklist(
271                             id="cl-ctrl-core-all",
272                             options=[{"label": "All", "value": "all"}, ],
273                             inline=True,
274                             switch=False
275                         ),
276                         dbc.Checklist(
277                             id="cl-ctrl-core",
278                             inline=True,
279                             switch=False
280                         )
281                     ]
282                 ),
283                 dbc.Row(
284                     id="row-ctrl-framesize",
285                     className="g-0",
286                     children=[
287                         dbc.Label("Frame Size"),
288                         dbc.Checklist(
289                             id="cl-ctrl-framesize-all",
290                             options=[{"label": "All", "value": "all"}, ],
291                             inline=True,
292                             switch=False
293                         ),
294                         dbc.Checklist(
295                             id="cl-ctrl-framesize",
296                             inline=True,
297                             switch=False
298                         )
299                     ]
300                 ),
301                 dbc.Row(
302                     id="row-ctrl-testtype",
303                     className="g-0",
304                     children=[
305                         dbc.Label("Test Type"),
306                         dbc.Checklist(
307                             id="cl-ctrl-testtype-all",
308                             options=[{"label": "All", "value": "all"}, ],
309                             inline=True,
310                             switch=False
311                         ),
312                         dbc.Checklist(
313                             id="cl-ctrl-testtype",
314                             inline=True,
315                             switch=False
316                         )
317                     ]
318                 ),
319                 dbc.Row(
320                     className="g-0",
321                     children=[
322                         dbc.Button(
323                             id="btn-ctrl-add",
324                             children="Add",
325                         )
326                     ]
327                 ),
328                 dbc.Row(
329                     className="g-0",
330                     children=[
331                         dcc.DatePickerRange(
332                             id="dpr-period",
333                             min_date_allowed=\
334                                 datetime.utcnow()-timedelta(days=180),
335                             max_date_allowed=datetime.utcnow(),
336                             initial_visible_month=datetime.utcnow(),
337                             start_date=datetime.utcnow() - timedelta(days=180),
338                             end_date=datetime.utcnow(),
339                             display_format="D MMMM YY"
340                         )
341                     ]
342                 )
343             ]
344         )
345
346     def _add_ctrl_shown(self) -> dbc.Row:
347         """
348         """
349         return dbc.Row(
350             id="div-ctrl-shown",
351             className="g-0",
352             children=[
353                 dbc.Row(
354                     className="g-0",
355                     children=[
356                         dbc.Label("Selected tests"),
357                         dbc.Checklist(
358                             id="cl-selected",
359                             options=[],
360                             inline=False
361                         )
362                     ]
363                 ),
364                 dbc.Row(
365                     className="g-0",
366                     children=[
367                         dbc.ButtonGroup(
368                             [
369                                 dbc.Button(
370                                     id="btn-sel-remove",
371                                     children="Remove Selected",
372                                     color="secondary",
373                                     disabled=False
374                                 ),
375                                 dbc.Button(
376                                     id="btn-sel-remove-all",
377                                     children="Remove All",
378                                     color="secondary",
379                                     disabled=False
380                                 ),
381                                 dbc.Button(
382                                     id="btn-sel-display",
383                                     children="Display",
384                                     color="secondary",
385                                     disabled=False
386                                 )
387                             ],
388                             size="md",
389                             className="me-1",
390                         ),
391                     ]
392                 )
393             ]
394         )
395
396     def callbacks(self, app):
397
398         @app.callback(
399             Output("dd-ctrl-area", "options"),
400             Output("dd-ctrl-area", "disabled"),
401             Input("dd-ctrl-phy", "value"),
402         )
403         def _update_dd_area(phy):
404             """
405             """
406
407             if phy is None:
408                 raise PreventUpdate
409
410             try:
411                 options = [
412                     {"label": self.spec_tbs[phy][v]["label"], "value": v}
413                         for v in [v for v in self.spec_tbs[phy].keys()]
414                 ]
415                 disable = False
416             except KeyError:
417                 options = list()
418                 disable = True
419
420             return options, disable
421
422         @app.callback(
423             Output("dd-ctrl-test", "options"),
424             Output("dd-ctrl-test", "disabled"),
425             State("dd-ctrl-phy", "value"),
426             Input("dd-ctrl-area", "value"),
427         )
428         def _update_dd_test(phy, area):
429             """
430             """
431
432             if not area:
433                 raise PreventUpdate
434
435             try:
436                 options = [
437                     {"label": v, "value": v}
438                         for v in self.spec_tbs[phy][area]["test"]
439                 ]
440                 disable = False
441             except KeyError:
442                 options = list()
443                 disable = True
444
445             return options, disable
446
447         @app.callback(
448             # Output("row-ctrl-core", "style"),
449             Output("cl-ctrl-core", "options"),
450             # Output("row-ctrl-framesize", "style"),
451             Output("cl-ctrl-framesize", "options"),
452             # Output("row-ctrl-testtype", "style"),
453             Output("cl-ctrl-testtype", "options"),
454             # Output("btn-ctrl-add", "disabled"),
455             State("dd-ctrl-phy", "value"),
456             State("dd-ctrl-area", "value"),
457             Input("dd-ctrl-test", "value"),
458         )
459         def _update_btn_add(phy, area, test):
460             """
461             """
462
463             if test is None:
464                 raise PreventUpdate
465
466             # core_style = {"display": "none"}
467             core_opts = []
468             # framesize_style = {"display": "none"}
469             framesize_opts = []
470             # testtype_style = {"display": "none"}
471             testtype_opts = []
472             # add_disabled = True
473             if phy and area and test:
474                 # core_style = {"display": "block"}
475                 core_opts = [
476                     {"label": v, "value": v}
477                         for v in self.spec_tbs[phy][area]["core"]
478                 ]
479                 # framesize_style = {"display": "block"}
480                 framesize_opts = [
481                     {"label": v, "value": v}
482                         for v in self.spec_tbs[phy][area]["frame-size"]
483                 ]
484                 # testtype_style = {"display": "block"}
485                 testtype_opts = [
486                     {"label": v, "value": v}
487                         for v in self.spec_tbs[phy][area]["test-type"]
488                 ]
489                 # add_disabled = False
490
491             return (
492                 # core_style,
493                 core_opts,
494                 # framesize_style,
495                 framesize_opts,
496                 # testtype_style,
497                 testtype_opts,
498                 # add_disabled
499             )
500
501         def _sync_checklists(opt, sel, all, id):
502             """
503             """
504             options = {v["value"] for v in opt}
505             input_id = callback_context.triggered[0]["prop_id"].split(".")[0]
506             if input_id == id:
507                 all = ["all"] if set(sel) == options else list()
508             else:
509                 sel = list(options) if all else list()
510             return sel, all
511
512         @app.callback(
513             Output("cl-ctrl-core", "value"),
514             Output("cl-ctrl-core-all", "value"),
515             State("cl-ctrl-core", "options"),
516             Input("cl-ctrl-core", "value"),
517             Input("cl-ctrl-core-all", "value"),
518             prevent_initial_call=True
519         )
520         def _sync_cl_core(opt, sel, all):
521             return _sync_checklists(opt, sel, all, "cl-ctrl-core")
522
523         @app.callback(
524             Output("cl-ctrl-framesize", "value"),
525             Output("cl-ctrl-framesize-all", "value"),
526             State("cl-ctrl-framesize", "options"),
527             Input("cl-ctrl-framesize", "value"),
528             Input("cl-ctrl-framesize-all", "value"),
529             prevent_initial_call=True
530         )
531         def _sync_cl_framesize(opt, sel, all):
532             return _sync_checklists(opt, sel, all, "cl-ctrl-framesize")
533
534         @app.callback(
535             Output("cl-ctrl-testtype", "value"),
536             Output("cl-ctrl-testtype-all", "value"),
537             State("cl-ctrl-testtype", "options"),
538             Input("cl-ctrl-testtype", "value"),
539             Input("cl-ctrl-testtype-all", "value"),
540             prevent_initial_call=True
541         )
542         def _sync_cl_testtype(opt, sel, all):
543             return _sync_checklists(opt, sel, all, "cl-ctrl-testtype")
544
545         @app.callback(
546             # Output("graph-tput", "figure"),
547             # Output("graph-latency", "figure"),
548             # Output("div-tput", "style"),
549             # Output("div-latency", "style"),
550             # Output("div-lat-metadata", "style"),
551             # Output("div-download", "style"),
552             Output("selected-tests", "data"),  # Store
553             Output("cl-selected", "options"),  # User selection
554             Output("dd-ctrl-phy", "value"),
555             Output("dd-ctrl-area", "value"),
556             Output("dd-ctrl-test", "value"),
557             State("selected-tests", "data"),  # Store
558             State("cl-selected", "value"),
559             State("dd-ctrl-phy", "value"),
560             State("dd-ctrl-area", "value"),
561             State("dd-ctrl-test", "value"),
562             State("cl-ctrl-core", "value"),
563             State("cl-ctrl-framesize", "value"),
564             State("cl-ctrl-testtype", "value"),
565             Input("btn-ctrl-add", "n_clicks"),
566             Input("btn-sel-display", "n_clicks"),
567             Input("btn-sel-remove", "n_clicks"),
568             Input("btn-sel-remove-all", "n_clicks"),
569             Input("dpr-period", "start_date"),
570             Input("dpr-period", "end_date"),
571             prevent_initial_call=True
572         )
573         def _process_list(store_sel, list_sel, phy, area, test, cores,
574                 framesizes, testtypes, btn_add, btn_display, btn_remove,
575                 btn_remove_all, d_start, d_end):
576             """
577             """
578
579             if not (btn_add or btn_display or btn_remove or btn_remove_all or \
580                     d_start or d_end):
581                 raise PreventUpdate
582
583             def _list_tests():
584                 # Display selected tests with checkboxes:
585                 if store_sel:
586                     return [
587                         {"label": v["id"], "value": v["id"]} for v in store_sel
588                     ]
589                 else:
590                     return list()
591
592             class RetunValue:
593                 def __init__(self) -> None:
594                     self._output = {
595                         # "graph-tput-figure": no_update,
596                         # "graph-lat-figure": no_update,
597                         # "div-tput-style": no_update,
598                         # "div-latency-style": no_update,
599                         # "div-lat-metadata-style": no_update,
600                         # "div-download-style": no_update,
601                         "selected-tests-data": no_update,
602                         "cl-selected-options": no_update,
603                         "dd-ctrl-phy-value": no_update,
604                         "dd-ctrl-area-value": no_update,
605                         "dd-ctrl-test-value": no_update,
606                     }
607
608                 def value(self):
609                     return tuple(self._output.values())
610
611                 def set_values(self, kwargs: dict) -> None:
612                     for key, val in kwargs.items():
613                         if key in self._output:
614                             self._output[key] = val
615                         else:
616                             raise KeyError(f"The key {key} is not defined.")
617
618
619             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
620
621             d_start = datetime(int(d_start[0:4]), int(d_start[5:7]),
622                 int(d_start[8:10]))
623             d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10]))
624
625             output = RetunValue()
626
627             if trigger_id == "btn-ctrl-add":
628                 # Add selected test to the list of tests in store:
629                 if phy and area and test and cores and framesizes and testtypes:
630                     if store_sel is None:
631                         store_sel = list()
632                     for core in cores:
633                         for framesize in framesizes:
634                             for ttype in testtypes:
635                                 tid = (
636                                     f"{phy.replace('af_xdp', 'af-xdp')}-"
637                                     f"{area}-"
638                                     f"{framesize.lower()}-"
639                                     f"{core.lower()}-"
640                                     f"{test}-"
641                                     f"{ttype.lower()}"
642                                 )
643                                 if tid not in [itm["id"] for itm in store_sel]:
644                                     store_sel.append({
645                                         "id": tid,
646                                         "phy": phy,
647                                         "area": area,
648                                         "test": test,
649                                         "framesize": framesize.lower(),
650                                         "core": core.lower(),
651                                         "testtype": ttype.lower()
652                                     })
653                 output.set_values({
654                     "selected-tests-data": store_sel,
655                     "cl-selected-options": _list_tests(),
656                     "dd-ctrl-phy-value": None,
657                     "dd-ctrl-area-value": None,
658                     "dd-ctrl-test-value": None,
659                 })
660
661             elif trigger_id in ("btn-sel-display", "dpr-period"):
662                 fig_tput, fig_lat = graph_trending(
663                     self.data, store_sel, self.layout, d_start, d_end
664                 )
665                 # output.set_values({
666                 #     "graph-tput-figure": \
667                 #         fig_tput if fig_tput else self.NO_GRAPH,
668                 #     "graph-lat-figure": \
669                 #         fig_lat if fig_lat else self.NO_GRAPH,
670                 #     "div-tput-style": \
671                 #         self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
672                 #     "div-latency-style": \
673                 #         self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
674                 #     "div-lat-metadata-style": \
675                 #         self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
676                 #     "div-download-style": \
677                 #         self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
678                 # })
679             elif trigger_id == "btn-sel-remove-all":
680                 output.set_values({
681                     "selected-tests-data": list(),
682                     "cl-selected-options": list()
683                 })
684             elif trigger_id == "btn-sel-remove":
685                 if list_sel:
686                     new_store_sel = list()
687                     for item in store_sel:
688                         if item["id"] not in list_sel:
689                             new_store_sel.append(item)
690                     store_sel = new_store_sel
691                 if store_sel:
692                     fig_tput, fig_lat = graph_trending(
693                         self.data, store_sel, self.layout, d_start, d_end
694                     )
695                     output.set_values({
696                         # "graph-tput-figure": \
697                         #     fig_tput if fig_tput else self.NO_GRAPH,
698                         # "graph-lat-figure": \
699                         #     fig_lat if fig_lat else self.NO_GRAPH,
700                         # "div-tput-style": \
701                         #     self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
702                         # "div-latency-style": \
703                         #     self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
704                         # "div-lat-metadata-style": \
705                         #     self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
706                         # "div-download-style": \
707                         #     self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
708                         "selected-tests-data": store_sel,
709                         "cl-selected-options": _list_tests()
710                     })
711                 else:
712                     output.set_values({
713                         # "graph-tput-figure": self.NO_GRAPH,
714                         # "graph-lat-figure": self.NO_GRAPH,
715                         # "div-tput-style": self.STYLE_HIDEN,
716                         # "div-latency-style": self.STYLE_HIDEN,
717                         # "div-lat-metadata-style": self.STYLE_HIDEN,
718                         # "div-download-style": self.STYLE_HIDEN,
719                         "selected-tests-data": store_sel,
720                         "cl-selected-options": _list_tests()
721                     })
722
723             return output.value()
724
725         # @app.callback(
726         #     Output("tput-metadata", "children"),
727         #     Input("graph-tput", "clickData")
728         # )
729         # def _show_tput_metadata(hover_data):
730         #     """
731         #     """
732         #     if not hover_data:
733         #         raise PreventUpdate
734
735         #     return hover_data["points"][0]["text"].replace("<br>", "\n")
736
737         # @app.callback(
738         #     Output("graph-latency-hdrh", "figure"),
739         #     Output("graph-latency-hdrh", "style"),
740         #     Output("lat-metadata", "children"),
741         #     Input("graph-latency", "clickData")
742         # )
743         # def _show_latency_hdhr(hover_data):
744         #     """
745         #     """
746         #     if not hover_data:
747         #         raise PreventUpdate
748
749         #     graph = no_update
750         #     hdrh_data = hover_data["points"][0].get("customdata", None)
751         #     if hdrh_data:
752         #         graph = graph_hdrh_latency(hdrh_data, self.layout)
753
754         #     return (
755         #         graph,
756         #         self.STYLE_INLINE,
757         #         hover_data["points"][0]["text"].replace("<br>", "\n")
758         #     )
759
760         # @app.callback(
761         #     Output("download-data", "data"),
762         #     State("selected-tests", "data"),
763         #     Input("btn-download-data", "n_clicks"),
764         #     prevent_initial_call=True
765         # )
766         # def _download_data(store_sel, n_clicks):
767         #     """
768         #     """
769
770         #     if not n_clicks:
771         #         raise PreventUpdate
772
773         #     df = pd.DataFrame()
774         #     for itm in store_sel:
775         #         sel_data = select_trending_data(self.data, itm)
776         #         if sel_data is None:
777         #             continue
778         #         df = pd.concat([df, sel_data], ignore_index=True)
779
780         #     return dcc.send_data_frame(df.to_csv, "trending_data.csv")