resources/tools/dash/app/pal/trending/layout.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17
  18 import pandas as pd
  19
  20 from dash import dcc
  21 from dash import html
  22 from dash import callback_context, no_update
  23 from dash import Input, Output, State, callback
  24 from dash.exceptions import PreventUpdate
  25 from yaml import load, FullLoader, YAMLError
  26 from datetime import datetime, timedelta
  27
  28 from ..data.data import Data
  29 from .graphs import graph_trending_tput, graph_hdrh_latency
  30
  31
  32 class Layout:
  33     """
  34     """
  35
  36     STYLE_HIDEN = {"display": "none"}
  37     STYLE_BLOCK = {"display": "block", "vertical-align": "top"}
  38     STYLE_INLINE ={
  39         "display": "inline-block",
  40         "vertical-align": "top",
  41         "width": "33%"
  42     }
  43     NO_GRAPH = {"data": [], "layout": {}, "frames": []}
  44
  45     def __init__(self, app, html_layout_file, spec_file, graph_layout_file,
  46         data_spec_file):
  47         """
  48         """
  49
  50         # Inputs
  51         self._app = app
  52         self._html_layout_file = html_layout_file
  53         self._spec_file = spec_file
  54         self._graph_layout_file = graph_layout_file
  55         self._data_spec_file = data_spec_file
  56
  57         # Read the data:
  58         data_mrr = Data(
  59             data_spec_file=self._data_spec_file,
  60             debug=True
  61         ).read_trending_mrr()
  62
  63         data_ndrpdr = Data(
  64             data_spec_file=self._data_spec_file,
  65             debug=True
  66         ).read_trending_ndrpdr()
  67
  68         self._data = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
  69
  70         # Read from files:
  71         self._html_layout = ""
  72         self._spec_tbs = None
  73         self._graph_layout = None
  74
  75         try:
  76             with open(self._html_layout_file, "r") as file_read:
  77                 self._html_layout = file_read.read()
  78         except IOError as err:
  79             raise RuntimeError(
  80                 f"Not possible to open the file {self._html_layout_file}\n{err}"
  81             )
  82
  83         try:
  84             with open(self._spec_file, "r") as file_read:
  85                 self._spec_tbs = load(file_read, Loader=FullLoader)
  86         except IOError as err:
  87             raise RuntimeError(
  88                 f"Not possible to open the file {self._spec_file,}\n{err}"
  89             )
  90         except YAMLError as err:
  91             raise RuntimeError(
  92                 f"An error occurred while parsing the specification file "
  93                 f"{self._spec_file,}\n"
  94                 f"{err}"
  95             )
  96
  97         try:
  98             with open(self._graph_layout_file, "r") as file_read:
  99                 self._graph_layout = load(file_read, Loader=FullLoader)
 100         except IOError as err:
 101             raise RuntimeError(
 102                 f"Not possible to open the file {self._graph_layout_file}\n"
 103                 f"{err}"
 104             )
 105         except YAMLError as err:
 106             raise RuntimeError(
 107                 f"An error occurred while parsing the specification file "
 108                 f"{self._graph_layout_file}\n"
 109                 f"{err}"
 110             )
 111
 112         # Callbacks:
 113         if self._app is not None and hasattr(self, 'callbacks'):
 114             self.callbacks(self._app)
 115
 116     @property
 117     def html_layout(self):
 118         return self._html_layout
 119
 120     @property
 121     def spec_tbs(self):
 122         return self._spec_tbs
 123
 124     @property
 125     def data(self):
 126         return self._data
 127
 128     @property
 129     def layout(self):
 130         return self._graph_layout
 131
 132     def add_content(self):
 133         """
 134         """
 135         if self.html_layout and self.spec_tbs:
 136             return html.Div(
 137                 id="div-main",
 138                 children=[
 139                     dcc.Store(id="selected-tests"),
 140                     self._add_ctrl_div(),
 141                     self._add_plotting_div()
 142                 ]
 143             )
 144         else:
 145             return html.Div(
 146             id="div-main-error",
 147             children="An Error Occured."
 148         )
 149
 150     def _add_ctrl_div(self):
 151         """Add div with controls. It is placed on the left side.
 152         """
 153         return html.Div(
 154             id="div-controls",
 155             children=[
 156                 html.Div(
 157                     id="div-controls-tabs",
 158                     children=[
 159                         self._add_ctrl_select(),
 160                         self._add_ctrl_shown()
 161                     ]
 162                 )
 163             ],
 164             style={
 165                 "display": "inline-block",
 166                 "width": "18%",
 167                 "padding": "5px"
 168             }
 169         )
 170
 171     def _add_plotting_div(self):
 172         """Add div with plots and tables. It is placed on the right side.
 173         """
 174         return html.Div(
 175             id="div-plotting-area",
 176             children=[
 177                 dcc.Loading(
 178                     id="loading-graph",
 179                     children=[
 180                         dcc.Graph(
 181                             id="graph-tput",
 182                             style=self.STYLE_HIDEN
 183                         ),
 184                         dcc.Graph(
 185                             id="graph-latency",
 186                             style=self.STYLE_HIDEN
 187                         )
 188                     ],
 189                     type="circle"
 190                 ),
 191                 html.Div(
 192                     children=[
 193                         html.Div(
 194                             id="div-tput-metadata",
 195                             children=[
 196                                 dcc.Markdown("**Throughput**"),
 197                                 html.Pre(
 198                                     id="tput-metadata",
 199                                     children="Click on data points in the graph"
 200                                 )
 201                             ],
 202                             style=self.STYLE_HIDEN
 203                         ),
 204                         html.Div(
 205                             id="div-latency-metadata",
 206                             children=[
 207                                 dcc.Markdown("**Latency**"),
 208                                 html.Pre(
 209                                     id="latency-metadata",
 210                                     children="Click on data points in the graph"
 211                                 )
 212                             ],
 213                             style=self.STYLE_HIDEN
 214                         ),
 215                         html.Div(
 216                             id="div-latency-hdrh",
 217                             children=[
 218                                 dcc.Loading(
 219                                     id="loading-hdrh-latency-graph",
 220                                     children=[
 221                                         dcc.Graph(
 222                                             id="graph-latency-hdrh"
 223                                         )
 224                                     ],
 225                                     type="circle"
 226                                 )
 227                             ],
 228                             style=self.STYLE_HIDEN
 229                         )
 230                     ]
 231                 )
 232             ],
 233             style={
 234                 "vertical-align": "top",
 235                 "display": "inline-block",
 236                 "width": "80%",
 237                 "padding": "5px"
 238             }
 239         )
 240
 241     def _add_ctrl_shown(self):
 242         """
 243         """
 244         return html.Div(
 245             id="div-ctrl-shown",
 246             children=[
 247                 html.H5("Selected tests"),
 248                 html.Div(
 249                     id="container-selected-tests",
 250                     children=[
 251                         dcc.Checklist(
 252                             id="cl-selected",
 253                             options=[],
 254                             labelStyle={"display": "block"}
 255                         ),
 256                         html.Button(
 257                             id="btn-sel-remove",
 258                             children="Remove Selected",
 259                             disabled=False
 260                         ),
 261                         html.Button(
 262                             id="btn-sel-display",
 263                             children="Display",
 264                             disabled=False
 265                         )
 266                     ]
 267                 ),
 268             ]
 269         )
 270
 271     def _add_ctrl_select(self):
 272         """
 273         """
 274         return html.Div(
 275             id="div-ctrl-select",
 276             children=[
 277                 html.H5("Physical Test Bed Topology, NIC and Driver"),
 278                 dcc.Dropdown(
 279                     id="dd-ctrl-phy",
 280                     placeholder="Select a Physical Test Bed Topology...",
 281                     multi=False,
 282                     clearable=False,
 283                     options=[
 284                         {"label": k, "value": k} for k in self.spec_tbs.keys()
 285                     ],
 286                 ),
 287                 html.H5("Area"),
 288                 dcc.Dropdown(
 289                     id="dd-ctrl-area",
 290                     placeholder="Select an Area...",
 291                     disabled=True,
 292                     multi=False,
 293                     clearable=False,
 294                 ),
 295                 html.H5("Test"),
 296                 dcc.Dropdown(
 297                     id="dd-ctrl-test",
 298                     placeholder="Select a Test...",
 299                     disabled=True,
 300                     multi=False,
 301                     clearable=False,
 302                 ),
 303                 html.Div(
 304                     id="div-ctrl-core",
 305                     children=[
 306                         html.H5("Number of Cores"),
 307                         dcc.Checklist(
 308                             id="cl-ctrl-core-all",
 309                             options=[{"label": "All", "value": "all"}, ],
 310                             labelStyle={"display": "inline-block"}
 311                         ),
 312                         dcc.Checklist(
 313                             id="cl-ctrl-core",
 314                             labelStyle={"display": "inline-block"}
 315                         )
 316                     ],
 317                     style={"display": "none"}
 318                 ),
 319                 html.Div(
 320                     id="div-ctrl-framesize",
 321                     children=[
 322                         html.H5("Frame Size"),
 323                         dcc.Checklist(
 324                             id="cl-ctrl-framesize-all",
 325                             options=[{"label": "All", "value": "all"}, ],
 326                             labelStyle={"display": "inline-block"}
 327                         ),
 328                         dcc.Checklist(
 329                             id="cl-ctrl-framesize",
 330                             labelStyle={"display": "inline-block"}
 331                         )
 332                     ],
 333                     style={"display": "none"}
 334                 ),
 335                 html.Div(
 336                     id="div-ctrl-testtype",
 337                     children=[
 338                         html.H5("Test Type"),
 339                         dcc.Checklist(
 340                             id="cl-ctrl-testtype-all",
 341                             options=[{"label": "All", "value": "all"}, ],
 342                             labelStyle={"display": "inline-block"}
 343                         ),
 344                         dcc.Checklist(
 345                             id="cl-ctrl-testtype",
 346                             labelStyle={"display": "inline-block"}
 347                         )
 348                     ],
 349                     style={"display": "none"}
 350                 ),
 351                 html.Button(
 352                     id="btn-ctrl-add",
 353                     children="Add",
 354                     disabled=True
 355                 ),
 356                 html.Br(),
 357                 dcc.DatePickerRange(
 358                     id="dpr-period",
 359                     min_date_allowed=datetime.utcnow() - timedelta(days=180),
 360                     max_date_allowed=datetime.utcnow(),
 361                     initial_visible_month=datetime.utcnow(),
 362                     start_date=datetime.utcnow() - timedelta(days=180),
 363                     end_date=datetime.utcnow(),
 364                     display_format="D MMMM YY"
 365                 )
 366             ]
 367         )
 368
 369     def callbacks(self, app):
 370
 371         @app.callback(
 372             Output("dd-ctrl-area", "options"),
 373             Output("dd-ctrl-area", "disabled"),
 374             Input("dd-ctrl-phy", "value"),
 375         )
 376         def _update_dd_area(phy):
 377             """
 378             """
 379
 380             if phy is None:
 381                 raise PreventUpdate
 382
 383             try:
 384                 options = [
 385                     {"label": self.spec_tbs[phy][v]["label"], "value": v}
 386                         for v in [v for v in self.spec_tbs[phy].keys()]
 387                 ]
 388                 disable = False
 389             except KeyError:
 390                 options = list()
 391                 disable = True
 392
 393             return options, disable
 394
 395         @app.callback(
 396             Output("dd-ctrl-test", "options"),
 397             Output("dd-ctrl-test", "disabled"),
 398             State("dd-ctrl-phy", "value"),
 399             Input("dd-ctrl-area", "value"),
 400         )
 401         def _update_dd_test(phy, area):
 402             """
 403             """
 404
 405             if not area:
 406                 raise PreventUpdate
 407
 408             try:
 409                 options = [
 410                     {"label": v, "value": v}
 411                         for v in self.spec_tbs[phy][area]["test"]
 412                 ]
 413                 disable = False
 414             except KeyError:
 415                 options = list()
 416                 disable = True
 417
 418             return options, disable
 419
 420         @app.callback(
 421             Output("div-ctrl-core", "style"),
 422             Output("cl-ctrl-core", "options"),
 423             Output("div-ctrl-framesize", "style"),
 424             Output("cl-ctrl-framesize", "options"),
 425             Output("div-ctrl-testtype", "style"),
 426             Output("cl-ctrl-testtype", "options"),
 427             Output("btn-ctrl-add", "disabled"),
 428             State("dd-ctrl-phy", "value"),
 429             State("dd-ctrl-area", "value"),
 430             Input("dd-ctrl-test", "value"),
 431         )
 432         def _update_btn_add(phy, area, test):
 433             """
 434             """
 435
 436             if test is None:
 437                 raise PreventUpdate
 438
 439             core_style = {"display": "none"}
 440             core_opts = []
 441             framesize_style = {"display": "none"}
 442             framesize_opts = []
 443             testtype_style = {"display": "none"}
 444             testtype_opts = []
 445             add_disabled = True
 446             if phy and area and test:
 447                 core_style = {"display": "block"}
 448                 core_opts = [
 449                     {"label": v, "value": v}
 450                         for v in self.spec_tbs[phy][area]["core"]
 451                 ]
 452                 framesize_style = {"display": "block"}
 453                 framesize_opts = [
 454                     {"label": v, "value": v}
 455                         for v in self.spec_tbs[phy][area]["frame-size"]
 456                 ]
 457                 testtype_style = {"display": "block"}
 458                 testtype_opts = [
 459                     {"label": v, "value": v}
 460                         for v in self.spec_tbs[phy][area]["test-type"]
 461                 ]
 462                 add_disabled = False
 463
 464             return (
 465                 core_style, core_opts,
 466                 framesize_style, framesize_opts,
 467                 testtype_style, testtype_opts,
 468                 add_disabled
 469             )
 470
 471         def _sync_checklists(opt, sel, all, id):
 472             """
 473             """
 474             options = {v["value"] for v in opt}
 475             input_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 476             if input_id == id:
 477                 all = ["all"] if set(sel) == options else list()
 478             else:
 479                 sel = list(options) if all else list()
 480             return sel, all
 481
 482         @app.callback(
 483             Output("cl-ctrl-core", "value"),
 484             Output("cl-ctrl-core-all", "value"),
 485             State("cl-ctrl-core", "options"),
 486             Input("cl-ctrl-core", "value"),
 487             Input("cl-ctrl-core-all", "value"),
 488             prevent_initial_call=True
 489         )
 490         def _sync_cl_core(opt, sel, all):
 491             return _sync_checklists(opt, sel, all, "cl-ctrl-core")
 492
 493         @app.callback(
 494             Output("cl-ctrl-framesize", "value"),
 495             Output("cl-ctrl-framesize-all", "value"),
 496             State("cl-ctrl-framesize", "options"),
 497             Input("cl-ctrl-framesize", "value"),
 498             Input("cl-ctrl-framesize-all", "value"),
 499             prevent_initial_call=True
 500         )
 501         def _sync_cl_framesize(opt, sel, all):
 502             return _sync_checklists(opt, sel, all, "cl-ctrl-framesize")
 503
 504         @app.callback(
 505             Output("cl-ctrl-testtype", "value"),
 506             Output("cl-ctrl-testtype-all", "value"),
 507             State("cl-ctrl-testtype", "options"),
 508             Input("cl-ctrl-testtype", "value"),
 509             Input("cl-ctrl-testtype-all", "value"),
 510             prevent_initial_call=True
 511         )
 512         def _sync_cl_testtype(opt, sel, all):
 513             return _sync_checklists(opt, sel, all, "cl-ctrl-testtype")
 514
 515         @app.callback(
 516             Output("graph-tput", "figure"),
 517             Output("graph-tput", "style"),
 518             Output("div-tput-metadata", "style"),
 519             Output("graph-latency", "figure"),
 520             Output("graph-latency", "style"),
 521             Output("div-latency-metadata", "style"),
 522             Output("selected-tests", "data"),  # Store
 523             Output("cl-selected", "options"),  # User selection
 524             Output("dd-ctrl-phy", "value"),
 525             Output("dd-ctrl-area", "value"),
 526             Output("dd-ctrl-test", "value"),
 527             State("selected-tests", "data"),  # Store
 528             State("cl-selected", "value"),
 529             State("dd-ctrl-phy", "value"),
 530             State("dd-ctrl-area", "value"),
 531             State("dd-ctrl-test", "value"),
 532             State("cl-ctrl-core", "value"),
 533             State("cl-ctrl-framesize", "value"),
 534             State("cl-ctrl-testtype", "value"),
 535             Input("btn-ctrl-add", "n_clicks"),
 536             Input("btn-sel-display", "n_clicks"),
 537             Input("btn-sel-remove", "n_clicks"),
 538             Input("dpr-period", "start_date"),
 539             Input("dpr-period", "end_date"),
 540             prevent_initial_call=True
 541         )
 542         def _process_list(store_sel, list_sel, phy, area, test, cores,
 543                 framesizes, testtypes, btn_add, btn_display, btn_remove,
 544                 d_start, d_end):
 545             """
 546             """
 547
 548             if not (btn_add or btn_display or btn_remove or d_start or d_end):
 549                 raise PreventUpdate
 550
 551             def _list_tests():
 552                 # Display selected tests with checkboxes:
 553                 if store_sel:
 554                     return [
 555                         {"label": v["id"], "value": v["id"]} for v in store_sel
 556                     ]
 557                 else:
 558                     return list()
 559
 560             class RetunValue:
 561                 def __init__(self) -> None:
 562                     self._output = {
 563                         "graph-tput-figure": no_update,
 564                         "graph-tput-style": no_update,
 565                         "div-tput-metadata-style": no_update,
 566                         "graph-lat-figure": no_update,
 567                         "graph-lat-style": no_update,
 568                         "div-lat-metadata-style": no_update,
 569                         "selected-tests-data": no_update,
 570                         "cl-selected-options": no_update,
 571                         "dd-ctrl-phy-value": no_update,
 572                         "dd-ctrl-area-value": no_update,
 573                         "dd-ctrl-test-value": no_update,
 574                     }
 575
 576                 def value(self):
 577                     return tuple(self._output.values())
 578
 579                 def set_values(self, kwargs: dict) -> None:
 580                     for key, val in kwargs.items():
 581                         if key in self._output:
 582                             self._output[key] = val
 583                         else:
 584                             raise KeyError(f"The key {key} is not defined.")
 585
 586
 587             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 588
 589             d_start = datetime(int(d_start[0:4]), int(d_start[5:7]),
 590                 int(d_start[8:10]))
 591             d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10]))
 592
 593             output = RetunValue()
 594
 595             if trigger_id == "btn-ctrl-add":
 596                 # Add selected test to the list of tests in store:
 597                 if phy and area and test and cores and framesizes and testtypes:
 598                     if store_sel is None:
 599                         store_sel = list()
 600                     for core in cores:
 601                         for framesize in framesizes:
 602                             for ttype in testtypes:
 603                                 tid = (
 604                                     f"{phy.replace('af_xdp', 'af-xdp')}-"
 605                                     f"{area}-"
 606                                     f"{framesize.lower()}-"
 607                                     f"{core.lower()}-"
 608                                     f"{test}-"
 609                                     f"{ttype.lower()}"
 610                                 )
 611                                 if tid not in [itm["id"] for itm in store_sel]:
 612                                     store_sel.append({
 613                                         "id": tid,
 614                                         "phy": phy,
 615                                         "area": area,
 616                                         "test": test,
 617                                         "framesize": framesize.lower(),
 618                                         "core": core.lower(),
 619                                         "testtype": ttype.lower()
 620                                     })
 621                 output.set_values({
 622                     "selected-tests-data": store_sel,
 623                     "cl-selected-options": _list_tests(),
 624                     "dd-ctrl-phy-value": None,
 625                     "dd-ctrl-area-value": None,
 626                     "dd-ctrl-test-value": None,
 627                 })
 628
 629             elif trigger_id in ("btn-sel-display", "dpr-period"):
 630                 fig_tput, fig_lat = graph_trending_tput(
 631                     self.data, store_sel, self.layout, d_start, d_end
 632                 )
 633                 output.set_values({
 634                     "graph-tput-figure": \
 635                         fig_tput if fig_tput else self.NO_GRAPH,
 636                     "graph-tput-style": \
 637                         self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
 638                     "div-tput-metadata-style": \
 639                         self.STYLE_INLINE if fig_tput else self.STYLE_HIDEN,
 640                     "graph-lat-figure": \
 641                         fig_lat if fig_lat else self.NO_GRAPH,
 642                     "graph-lat-style": \
 643                         self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
 644                     "div-lat-metadata-style": \
 645                         self.STYLE_INLINE if fig_lat else self.STYLE_HIDEN
 646                 })
 647
 648             elif trigger_id == "btn-sel-remove":
 649                 if list_sel:
 650                     new_store_sel = list()
 651                     for item in store_sel:
 652                         if item["id"] not in list_sel:
 653                             new_store_sel.append(item)
 654                     store_sel = new_store_sel
 655                 if store_sel:
 656                     fig_tput, fig_lat = graph_trending_tput(
 657                         self.data, store_sel, self.layout, d_start, d_end
 658                     )
 659                     output.set_values({
 660                         "graph-tput-figure": \
 661                             fig_tput if fig_tput else self.NO_GRAPH,
 662                         "graph-tput-style": \
 663                             self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
 664                         "div-tput-metadata-style": \
 665                             self.STYLE_INLINE if fig_tput else self.STYLE_HIDEN,
 666                         "graph-lat-figure": \
 667                             fig_lat if fig_lat else self.NO_GRAPH,
 668                         "graph-lat-style": \
 669                             self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
 670                         "div-lat-metadata-style": \
 671                             self.STYLE_INLINE if fig_lat else self.STYLE_HIDEN,
 672                         "selected-tests-data": store_sel,
 673                         "cl-selected-options": _list_tests()
 674                     })
 675                 else:
 676                     output.set_values({
 677                         "graph-tput-figure": self.NO_GRAPH,
 678                         "graph-tput-style": self.STYLE_HIDEN,
 679                         "div-tput-metadata-style": self.STYLE_HIDEN,
 680                         "graph-lat-figure": self.NO_GRAPH,
 681                         "graph-lat-style": self.STYLE_HIDEN,
 682                         "div-lat-metadata-style": self.STYLE_HIDEN,
 683                         "selected-tests-data": store_sel,
 684                         "cl-selected-options": _list_tests()
 685                     })
 686
 687             return output.value()
 688
 689         @app.callback(
 690             Output("tput-metadata", "children"),
 691             Input("graph-tput", "clickData")
 692         )
 693         def _show_tput_metadata(hover_data):
 694             if not hover_data:
 695                 raise PreventUpdate
 696             return hover_data["points"][0]["text"].replace("<br>", "\n"),
 697
 698         @app.callback(
 699             Output("latency-metadata", "children"),
 700             Output("graph-latency-hdrh", "figure"),
 701             Output("div-latency-hdrh", "style"),
 702             Input("graph-latency", "clickData")
 703         )
 704         def _show_latency_metadata(hover_data):
 705             if not hover_data:
 706                 raise PreventUpdate
 707             graph = graph_hdrh_latency(
 708                 hover_data["points"][0]["customdata"], self.layout
 709             )
 710             if not graph:
 711                 graph = no_update
 712             return (
 713                 hover_data["points"][0]["text"].replace("<br>", "\n"),
 714                 graph,
 715                 self.STYLE_INLINE if graph else self.STYLE_HIDEN
 716             )