resources/tools/dash/app/pal/trending/layout.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17
  18 import json
  19 import pandas as pd
  20
  21 from dash import dcc
  22 from dash import html
  23 from dash import callback_context, no_update
  24 from dash import Input, Output, State, callback
  25 from dash.exceptions import PreventUpdate
  26 from yaml import load, FullLoader, YAMLError
  27 from datetime import datetime, timedelta
  28
  29 from ..data.data import Data
  30 from .graphs import trending_tput
  31
  32
  33 class Layout:
  34     """
  35     """
  36
  37     STYLE_HIDEN = {"display": "none"}
  38     STYLE_BLOCK = {"display": "block"}
  39     STYLE_INLINE ={"display": "inline-block", "width": "50%"}
  40     NO_GRAPH = {"data": [], "layout": {}, "frames": []}
  41
  42     def __init__(self, app, html_layout_file, spec_file, graph_layout_file,
  43         data_spec_file):
  44         """
  45         """
  46
  47         # Inputs
  48         self._app = app
  49         self._html_layout_file = html_layout_file
  50         self._spec_file = spec_file
  51         self._graph_layout_file = graph_layout_file
  52         self._data_spec_file = data_spec_file
  53
  54         # Read the data:
  55         data_mrr = Data(
  56             data_spec_file=self._data_spec_file,
  57             debug=True
  58         ).read_trending_mrr()
  59
  60         data_ndrpdr = Data(
  61             data_spec_file=self._data_spec_file,
  62             debug=True
  63         ).read_trending_ndrpdr()
  64
  65         self._data = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
  66
  67         # Read from files:
  68         self._html_layout = ""
  69         self._spec_tbs = None
  70         self._graph_layout = None
  71
  72         try:
  73             with open(self._html_layout_file, "r") as file_read:
  74                 self._html_layout = file_read.read()
  75         except IOError as err:
  76             raise RuntimeError(
  77                 f"Not possible to open the file {self._html_layout_file}\n{err}"
  78             )
  79
  80         try:
  81             with open(self._spec_file, "r") as file_read:
  82                 self._spec_tbs = load(file_read, Loader=FullLoader)
  83         except IOError as err:
  84             raise RuntimeError(
  85                 f"Not possible to open the file {self._spec_file,}\n{err}"
  86             )
  87         except YAMLError as err:
  88             raise RuntimeError(
  89                 f"An error occurred while parsing the specification file "
  90                 f"{self._spec_file,}\n"
  91                 f"{err}"
  92             )
  93
  94         try:
  95             with open(self._graph_layout_file, "r") as file_read:
  96                 self._graph_layout = load(file_read, Loader=FullLoader)
  97         except IOError as err:
  98             raise RuntimeError(
  99                 f"Not possible to open the file {self._graph_layout_file}\n"
 100                 f"{err}"
 101             )
 102         except YAMLError as err:
 103             raise RuntimeError(
 104                 f"An error occurred while parsing the specification file "
 105                 f"{self._graph_layout_file}\n"
 106                 f"{err}"
 107             )
 108
 109         # Callbacks:
 110         if self._app is not None and hasattr(self, 'callbacks'):
 111             self.callbacks(self._app)
 112
 113     @property
 114     def html_layout(self):
 115         return self._html_layout
 116
 117     @property
 118     def spec_tbs(self):
 119         return self._spec_tbs
 120
 121     @property
 122     def data(self):
 123         return self._data
 124
 125     @property
 126     def layout(self):
 127         return self._graph_layout
 128
 129     def add_content(self):
 130         """
 131         """
 132         if self.html_layout and self.spec_tbs:
 133             return html.Div(
 134                 id="div-main",
 135                 children=[
 136                     dcc.Store(id="selected-tests"),
 137                     self._add_ctrl_div(),
 138                     self._add_plotting_div()
 139                 ]
 140             )
 141         else:
 142             return html.Div(
 143             id="div-main-error",
 144             children="An Error Occured."
 145         )
 146
 147     def _add_ctrl_div(self):
 148         """Add div with controls. It is placed on the left side.
 149         """
 150         return html.Div(
 151             id="div-controls",
 152             children=[
 153                 html.Div(
 154                     id="div-controls-tabs",
 155                     children=[
 156                         self._add_ctrl_select(),
 157                         self._add_ctrl_shown()
 158                     ]
 159                 )
 160             ],
 161             style={
 162                 "display": "inline-block",
 163                 "width": "18%",
 164                 "padding": "5px"
 165             }
 166         )
 167
 168     def _add_plotting_div(self):
 169         """Add div with plots and tables. It is placed on the right side.
 170         """
 171         return html.Div(
 172             id="div-plotting-area",
 173             children=[
 174                 dcc.Loading(
 175                     id="loading-graph",
 176                     children=[
 177                         dcc.Graph(
 178                             id="graph-tput",
 179                             style=self.STYLE_HIDEN
 180                         ),
 181                         dcc.Graph(
 182                             id="graph-latency",
 183                             style=self.STYLE_HIDEN
 184                         )
 185                     ],
 186                     type="circle"
 187                 ),
 188                 html.Div(
 189                     children=[
 190                         html.Div(
 191                             id="div-tput-metadata",
 192                             children=[
 193                                 dcc.Markdown("""
 194                                 **Metadata**
 195
 196                                 Click on data points in the graph.
 197                                 """),
 198                                 html.Pre(
 199                                     id="tput-metadata"
 200                                 )
 201                             ],
 202                             style=self.STYLE_HIDEN
 203                         ),
 204                         html.Div(
 205                             id="div-latency-metadata",
 206                             children=[
 207                                 dcc.Markdown("""
 208                                 **Metadata**
 209
 210                                 Click on data points in the graph.
 211                                 """),
 212                                 html.Pre(
 213                                     id="latency-metadata"
 214                                 )
 215                             ],
 216                             style=self.STYLE_HIDEN
 217                         )
 218                     ]
 219                 )
 220             ],
 221             style={
 222                 "vertical-align": "top",
 223                 "display": "inline-block",
 224                 "width": "80%",
 225                 "padding": "5px"
 226             }
 227         )
 228
 229     def _add_ctrl_shown(self):
 230         """
 231         """
 232         return html.Div(
 233             id="div-ctrl-shown",
 234             children=[
 235                 html.H5("Selected tests"),
 236                 html.Div(
 237                     id="container-selected-tests",
 238                     children=[
 239                         dcc.Checklist(
 240                             id="cl-selected",
 241                             options=[],
 242                             labelStyle={"display": "block"}
 243                         ),
 244                         html.Button(
 245                             id="btn-sel-remove",
 246                             children="Remove Selected",
 247                             disabled=False
 248                         ),
 249                         html.Button(
 250                             id="btn-sel-display",
 251                             children="Display",
 252                             disabled=False
 253                         )
 254                     ]
 255                 ),
 256             ]
 257         )
 258
 259     def _add_ctrl_select(self):
 260         """
 261         """
 262         return html.Div(
 263             id="div-ctrl-select",
 264             children=[
 265                 html.H5("Physical Test Bed Topology, NIC and Driver"),
 266                 dcc.Dropdown(
 267                     id="dd-ctrl-phy",
 268                     placeholder="Select a Physical Test Bed Topology...",
 269                     multi=False,
 270                     clearable=False,
 271                     options=[
 272                         {"label": k, "value": k} for k in self.spec_tbs.keys()
 273                     ],
 274                 ),
 275                 html.H5("Area"),
 276                 dcc.Dropdown(
 277                     id="dd-ctrl-area",
 278                     placeholder="Select an Area...",
 279                     disabled=True,
 280                     multi=False,
 281                     clearable=False,
 282                 ),
 283                 html.H5("Test"),
 284                 dcc.Dropdown(
 285                     id="dd-ctrl-test",
 286                     placeholder="Select a Test...",
 287                     disabled=True,
 288                     multi=False,
 289                     clearable=False,
 290                 ),
 291                 html.Div(
 292                     id="div-ctrl-core",
 293                     children=[
 294                         html.H5("Number of Cores"),
 295                         dcc.Checklist(
 296                             id="cl-ctrl-core-all",
 297                             options=[{"label": "All", "value": "all"}, ],
 298                             labelStyle={"display": "inline-block"}
 299                         ),
 300                         dcc.Checklist(
 301                             id="cl-ctrl-core",
 302                             labelStyle={"display": "inline-block"}
 303                         )
 304                     ],
 305                     style={"display": "none"}
 306                 ),
 307                 html.Div(
 308                     id="div-ctrl-framesize",
 309                     children=[
 310                         html.H5("Frame Size"),
 311                         dcc.Checklist(
 312                             id="cl-ctrl-framesize-all",
 313                             options=[{"label": "All", "value": "all"}, ],
 314                             labelStyle={"display": "inline-block"}
 315                         ),
 316                         dcc.Checklist(
 317                             id="cl-ctrl-framesize",
 318                             labelStyle={"display": "inline-block"}
 319                         )
 320                     ],
 321                     style={"display": "none"}
 322                 ),
 323                 html.Div(
 324                     id="div-ctrl-testtype",
 325                     children=[
 326                         html.H5("Test Type"),
 327                         dcc.Checklist(
 328                             id="cl-ctrl-testtype-all",
 329                             options=[{"label": "All", "value": "all"}, ],
 330                             labelStyle={"display": "inline-block"}
 331                         ),
 332                         dcc.Checklist(
 333                             id="cl-ctrl-testtype",
 334                             labelStyle={"display": "inline-block"}
 335                         )
 336                     ],
 337                     style={"display": "none"}
 338                 ),
 339                 html.Button(
 340                     id="btn-ctrl-add",
 341                     children="Add",
 342                     disabled=True
 343                 ),
 344                 html.Br(),
 345                 dcc.DatePickerRange(
 346                     id="dpr-period",
 347                     min_date_allowed=datetime.utcnow() - timedelta(days=180),
 348                     max_date_allowed=datetime.utcnow(),
 349                     initial_visible_month=datetime.utcnow(),
 350                     start_date=datetime.utcnow() - timedelta(days=180),
 351                     end_date=datetime.utcnow(),
 352                     display_format="D MMMM YY"
 353                 )
 354             ]
 355         )
 356
 357     def callbacks(self, app):
 358
 359         @app.callback(
 360             Output("dd-ctrl-area", "options"),
 361             Output("dd-ctrl-area", "disabled"),
 362             Input("dd-ctrl-phy", "value"),
 363         )
 364         def _update_dd_area(phy):
 365             """
 366             """
 367
 368             if phy is None:
 369                 raise PreventUpdate
 370
 371             try:
 372                 options = [
 373                     {"label": self.spec_tbs[phy][v]["label"], "value": v}
 374                         for v in [v for v in self.spec_tbs[phy].keys()]
 375                 ]
 376                 disable = False
 377             except KeyError:
 378                 options = list()
 379                 disable = True
 380
 381             return options, disable
 382
 383         @app.callback(
 384             Output("dd-ctrl-test", "options"),
 385             Output("dd-ctrl-test", "disabled"),
 386             State("dd-ctrl-phy", "value"),
 387             Input("dd-ctrl-area", "value"),
 388         )
 389         def _update_dd_test(phy, area):
 390             """
 391             """
 392
 393             if not area:
 394                 raise PreventUpdate
 395
 396             try:
 397                 options = [
 398                     {"label": v, "value": v}
 399                         for v in self.spec_tbs[phy][area]["test"]
 400                 ]
 401                 disable = False
 402             except KeyError:
 403                 options = list()
 404                 disable = True
 405
 406             return options, disable
 407
 408         @app.callback(
 409             Output("div-ctrl-core", "style"),
 410             Output("cl-ctrl-core", "options"),
 411             Output("div-ctrl-framesize", "style"),
 412             Output("cl-ctrl-framesize", "options"),
 413             Output("div-ctrl-testtype", "style"),
 414             Output("cl-ctrl-testtype", "options"),
 415             Output("btn-ctrl-add", "disabled"),
 416             State("dd-ctrl-phy", "value"),
 417             State("dd-ctrl-area", "value"),
 418             Input("dd-ctrl-test", "value"),
 419         )
 420         def _update_btn_add(phy, area, test):
 421             """
 422             """
 423
 424             if test is None:
 425                 raise PreventUpdate
 426
 427             core_style = {"display": "none"}
 428             core_opts = []
 429             framesize_style = {"display": "none"}
 430             framesize_opts = []
 431             testtype_style = {"display": "none"}
 432             testtype_opts = []
 433             add_disabled = True
 434             if phy and area and test:
 435                 core_style = {"display": "block"}
 436                 core_opts = [
 437                     {"label": v, "value": v}
 438                         for v in self.spec_tbs[phy][area]["core"]
 439                 ]
 440                 framesize_style = {"display": "block"}
 441                 framesize_opts = [
 442                     {"label": v, "value": v}
 443                         for v in self.spec_tbs[phy][area]["frame-size"]
 444                 ]
 445                 testtype_style = {"display": "block"}
 446                 testtype_opts = [
 447                     {"label": v, "value": v}
 448                         for v in self.spec_tbs[phy][area]["test-type"]
 449                 ]
 450                 add_disabled = False
 451
 452             return (
 453                 core_style, core_opts,
 454                 framesize_style, framesize_opts,
 455                 testtype_style, testtype_opts,
 456                 add_disabled
 457             )
 458
 459         def _sync_checklists(opt, sel, all, id):
 460             """
 461             """
 462             options = {v["value"] for v in opt}
 463             input_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 464             if input_id == id:
 465                 all = ["all"] if set(sel) == options else list()
 466             else:
 467                 sel = list(options) if all else list()
 468             return sel, all
 469
 470         @app.callback(
 471             Output("cl-ctrl-core", "value"),
 472             Output("cl-ctrl-core-all", "value"),
 473             State("cl-ctrl-core", "options"),
 474             Input("cl-ctrl-core", "value"),
 475             Input("cl-ctrl-core-all", "value"),
 476             prevent_initial_call=True
 477         )
 478         def _sync_cl_core(opt, sel, all):
 479             return _sync_checklists(opt, sel, all, "cl-ctrl-core")
 480
 481         @app.callback(
 482             Output("cl-ctrl-framesize", "value"),
 483             Output("cl-ctrl-framesize-all", "value"),
 484             State("cl-ctrl-framesize", "options"),
 485             Input("cl-ctrl-framesize", "value"),
 486             Input("cl-ctrl-framesize-all", "value"),
 487             prevent_initial_call=True
 488         )
 489         def _sync_cl_framesize(opt, sel, all):
 490             return _sync_checklists(opt, sel, all, "cl-ctrl-framesize")
 491
 492         @app.callback(
 493             Output("cl-ctrl-testtype", "value"),
 494             Output("cl-ctrl-testtype-all", "value"),
 495             State("cl-ctrl-testtype", "options"),
 496             Input("cl-ctrl-testtype", "value"),
 497             Input("cl-ctrl-testtype-all", "value"),
 498             prevent_initial_call=True
 499         )
 500         def _sync_cl_testtype(opt, sel, all):
 501             return _sync_checklists(opt, sel, all, "cl-ctrl-testtype")
 502
 503         @app.callback(
 504             Output("graph-tput", "figure"),
 505             Output("graph-tput", "style"),
 506             Output("div-tput-metadata", "style"),
 507             Output("graph-latency", "figure"),
 508             Output("graph-latency", "style"),
 509             Output("div-latency-metadata", "style"),
 510             Output("selected-tests", "data"),  # Store
 511             Output("cl-selected", "options"),  # User selection
 512             Output("dd-ctrl-phy", "value"),
 513             Output("dd-ctrl-area", "value"),
 514             Output("dd-ctrl-test", "value"),
 515             State("selected-tests", "data"),  # Store
 516             State("cl-selected", "value"),
 517             State("dd-ctrl-phy", "value"),
 518             State("dd-ctrl-area", "value"),
 519             State("dd-ctrl-test", "value"),
 520             State("cl-ctrl-core", "value"),
 521             State("cl-ctrl-framesize", "value"),
 522             State("cl-ctrl-testtype", "value"),
 523             Input("btn-ctrl-add", "n_clicks"),
 524             Input("btn-sel-display", "n_clicks"),
 525             Input("btn-sel-remove", "n_clicks"),
 526             Input("dpr-period", "start_date"),
 527             Input("dpr-period", "end_date"),
 528             prevent_initial_call=True
 529         )
 530         def _process_list(store_sel, list_sel, phy, area, test, cores,
 531                 framesizes, testtypes, btn_add, btn_display, btn_remove,
 532                 d_start, d_end):
 533             """
 534             """
 535
 536             if not (btn_add or btn_display or btn_remove or d_start or d_end):
 537                 raise PreventUpdate
 538
 539             def _list_tests():
 540                 # Display selected tests with checkboxes:
 541                 if store_sel:
 542                     return [
 543                         {"label": v["id"], "value": v["id"]} for v in store_sel
 544                     ]
 545                 else:
 546                     return list()
 547
 548             class RetunValue:
 549                 def __init__(self) -> None:
 550                     self._output = {
 551                         "graph-tput-figure": no_update,
 552                         "graph-tput-style": no_update,
 553                         "div-tput-metadata-style": no_update,
 554                         "graph-lat-figure": no_update,
 555                         "graph-lat-style": no_update,
 556                         "div-lat-metadata-style": no_update,
 557                         "selected-tests-data": no_update,
 558                         "cl-selected-options": no_update,
 559                         "dd-ctrl-phy-value": no_update,
 560                         "dd-ctrl-area-value": no_update,
 561                         "dd-ctrl-test-value": no_update,
 562                     }
 563
 564                 def value(self):
 565                     return tuple(self._output.values())
 566
 567                 def set_values(self, kwargs: dict) -> None:
 568                     for key, val in kwargs.items():
 569                         if key in self._output:
 570                             self._output[key] = val
 571                         else:
 572                             raise KeyError(f"The key {key} is not defined.")
 573
 574
 575             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 576
 577             d_start = datetime(int(d_start[0:4]), int(d_start[5:7]),
 578                 int(d_start[8:10]))
 579             d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10]))
 580
 581             output = RetunValue()
 582
 583             if trigger_id == "btn-ctrl-add":
 584                 # Add selected test to the list of tests in store:
 585                 if phy and area and test and cores and framesizes and testtypes:
 586                     if store_sel is None:
 587                         store_sel = list()
 588                     for core in cores:
 589                         for framesize in framesizes:
 590                             for ttype in testtypes:
 591                                 tid = (
 592                                     f"{phy.replace('af_xdp', 'af-xdp')}-"
 593                                     f"{area}-"
 594                                     f"{framesize.lower()}-"
 595                                     f"{core.lower()}-"
 596                                     f"{test}-"
 597                                     f"{ttype.lower()}"
 598                                 )
 599                                 if tid not in [itm["id"] for itm in store_sel]:
 600                                     store_sel.append({
 601                                         "id": tid,
 602                                         "phy": phy,
 603                                         "area": area,
 604                                         "test": test,
 605                                         "framesize": framesize.lower(),
 606                                         "core": core.lower(),
 607                                         "testtype": ttype.lower()
 608                                     })
 609                 output.set_values({
 610                     "selected-tests-data": store_sel,
 611                     "cl-selected-options": _list_tests(),
 612                     "dd-ctrl-phy-value": None,
 613                     "dd-ctrl-area-value": None,
 614                     "dd-ctrl-test-value": None,
 615                 })
 616
 617             elif trigger_id in ("btn-sel-display", "dpr-period"):
 618                 fig_tput, fig_lat = trending_tput(
 619                     self.data, store_sel, self.layout, d_start, d_end
 620                 )
 621                 output.set_values({
 622                     "graph-tput-figure": \
 623                         fig_tput if fig_tput else self.NO_GRAPH,
 624                     "graph-tput-style": \
 625                         self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
 626                     "div-tput-metadata-style": \
 627                         self.STYLE_INLINE if fig_tput else self.STYLE_HIDEN,
 628                     "graph-lat-figure": \
 629                         fig_lat if fig_lat else self.NO_GRAPH,
 630                     "graph-lat-style": \
 631                         self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
 632                     "div-lat-metadata-style": \
 633                         self.STYLE_INLINE if fig_lat else self.STYLE_HIDEN
 634                 })
 635
 636             elif trigger_id == "btn-sel-remove":
 637                 if list_sel:
 638                     new_store_sel = list()
 639                     for item in store_sel:
 640                         if item["id"] not in list_sel:
 641                             new_store_sel.append(item)
 642                     store_sel = new_store_sel
 643                 if store_sel:
 644                     fig_tput, fig_lat = trending_tput(
 645                         self.data, store_sel, self.layout, d_start, d_end
 646                     )
 647                     output.set_values({
 648                         "graph-tput-figure": \
 649                             fig_tput if fig_tput else self.NO_GRAPH,
 650                         "graph-tput-style": \
 651                             self.STYLE_BLOCK if fig_tput else self.STYLE_HIDEN,
 652                         "div-tput-metadata-style": \
 653                             self.STYLE_INLINE if fig_tput else self.STYLE_HIDEN,
 654                         "graph-lat-figure": \
 655                             fig_lat if fig_lat else self.NO_GRAPH,
 656                         "graph-lat-style": \
 657                             self.STYLE_BLOCK if fig_lat else self.STYLE_HIDEN,
 658                         "div-lat-metadata-style": \
 659                             self.STYLE_INLINE if fig_lat else self.STYLE_HIDEN,
 660                         "selected-tests-data": store_sel,
 661                         "cl-selected-options": _list_tests()
 662                     })
 663                 else:
 664                     output.set_values({
 665                         "graph-tput-figure": self.NO_GRAPH,
 666                         "graph-tput-style": self.STYLE_HIDEN,
 667                         "div-tput-metadata-style": self.STYLE_HIDEN,
 668                         "graph-lat-figure": self.NO_GRAPH,
 669                         "graph-lat-style": self.STYLE_HIDEN,
 670                         "div-lat-metadata-style": self.STYLE_HIDEN,
 671                         "selected-tests-data": store_sel,
 672                         "cl-selected-options": _list_tests()
 673                     })
 674
 675             return output.value()
 676
 677         @app.callback(
 678             Output("tput-metadata", "children"),
 679             Input("graph-tput", "clickData")
 680         )
 681         def _show_tput_metadata(hover_data):
 682             if not hover_data:
 683                 raise PreventUpdate
 684             return json.dumps(hover_data, indent=2)
 685
 686         @app.callback(
 687             Output("latency-metadata", "children"),
 688             Input("graph-latency", "clickData")
 689         )
 690         def _show_latency_metadata(hover_data):
 691             if not hover_data:
 692                 raise PreventUpdate
 693             return json.dumps(hover_data, indent=2)