csit.infra.dash/app/cdash/comparisons/layout.py

   1 # Copyright (c) 2024 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import pandas as pd
  18 import dash_bootstrap_components as dbc
  19
  20 from flask import Flask
  21 from dash import dcc, html, dash_table, callback_context, no_update, ALL
  22 from dash import Input, Output, State
  23 from dash.exceptions import PreventUpdate
  24 from dash.dash_table.Format import Format, Scheme
  25 from ast import literal_eval
  26
  27 from ..utils.constants import Constants as C
  28 from ..utils.control_panel import ControlPanel
  29 from ..utils.trigger import Trigger
  30 from ..utils.url_processing import url_decode
  31 from ..utils.utils import generate_options, gen_new_url, navbar_report, \
  32     filter_table_data
  33 from .tables import comparison_table
  34
  35
  36 # Control panel partameters and their default values.
  37 CP_PARAMS = {
  38     "dut-val": str(),
  39     "dutver-opt": list(),
  40     "dutver-dis": True,
  41     "dutver-val": str(),
  42     "infra-opt": list(),
  43     "infra-dis": True,
  44     "infra-val": str(),
  45     "core-opt": list(),
  46     "core-val": list(),
  47     "frmsize-opt": list(),
  48     "frmsize-val": list(),
  49     "ttype-opt": list(),
  50     "ttype-val": list(),
  51     "cmp-par-opt": list(),
  52     "cmp-par-dis": True,
  53     "cmp-par-val": str(),
  54     "cmp-val-opt": list(),
  55     "cmp-val-dis": True,
  56     "cmp-val-val": str(),
  57     "normalize-val": list(),
  58     "outliers-val": list()
  59 }
  60
  61 # List of comparable parameters.
  62 CMP_PARAMS = {
  63     "dutver": "Release and Version",
  64     "infra": "Infrastructure",
  65     "frmsize": "Frame Size",
  66     "core": "Number of Cores",
  67     "ttype": "Measurement"
  68 }
  69
  70
  71 class Layout:
  72     """The layout of the dash app and the callbacks.
  73     """
  74
  75     def __init__(
  76             self,
  77             app: Flask,
  78             data_iterative: pd.DataFrame,
  79             html_layout_file: str
  80         ) -> None:
  81         """Initialization:
  82         - save the input parameters,
  83         - prepare data for the control panel,
  84         - read HTML layout file,
  85
  86         :param app: Flask application running the dash application.
  87         :param data_iterative: Iterative data to be used in comparison tables.
  88         :param html_layout_file: Path and name of the file specifying the HTML
  89             layout of the dash application.
  90         :type app: Flask
  91         :type data_iterative: pandas.DataFrame
  92         :type html_layout_file: str
  93         """
  94
  95         # Inputs
  96         self._app = app
  97         self._html_layout_file = html_layout_file
  98         self._data = data_iterative
  99
 100         # Get structure of tests:
 101         tbs = dict()
 102         cols = [
 103             "job", "test_id", "test_type", "dut_type", "dut_version", "tg_type",
 104             "release", "passed"
 105         ]
 106         for _, row in self._data[cols].drop_duplicates().iterrows():
 107             lst_job = row["job"].split("-")
 108             dut = lst_job[1]
 109             dver = f"{row['release']}-{row['dut_version']}"
 110             tbed = "-".join(lst_job[-2:])
 111             lst_test_id = row["test_id"].split(".")
 112
 113             suite = lst_test_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
 114                 replace("2n-", "")
 115             test = lst_test_id[-1]
 116             nic = suite.split("-")[0]
 117             for driver in C.DRIVERS:
 118                 if driver in test:
 119                     drv = driver.replace("-", "_")
 120                     test = test.replace(f"{driver}-", "")
 121                     break
 122             else:
 123                 drv = "dpdk"
 124             infra = "-".join((tbed, nic, drv))
 125             lst_test = test.split("-")
 126             fsize = lst_test[0]
 127             core = lst_test[1] if lst_test[1] else "8C"
 128
 129             if tbs.get(dut, None) is None:
 130                 tbs[dut] = dict()
 131             if tbs[dut].get(dver, None) is None:
 132                 tbs[dut][dver] = dict()
 133             if tbs[dut][dver].get(infra, None) is None:
 134                 tbs[dut][dver][infra] = dict()
 135                 tbs[dut][dver][infra]["core"] = list()
 136                 tbs[dut][dver][infra]["fsize"] = list()
 137                 tbs[dut][dver][infra]["ttype"] = list()
 138             if core.upper() not in tbs[dut][dver][infra]["core"]:
 139                 tbs[dut][dver][infra]["core"].append(core.upper())
 140             if fsize.upper() not in tbs[dut][dver][infra]["fsize"]:
 141                 tbs[dut][dver][infra]["fsize"].append(fsize.upper())
 142             if row["test_type"] == "mrr":
 143                 if "MRR" not in tbs[dut][dver][infra]["ttype"]:
 144                     tbs[dut][dver][infra]["ttype"].append("MRR")
 145             elif row["test_type"] == "ndrpdr":
 146                 if "NDR" not in tbs[dut][dver][infra]["ttype"]:
 147                     tbs[dut][dver][infra]["ttype"].extend(
 148                         ("NDR", "PDR", "Latency")
 149                     )
 150             elif row["test_type"] == "hoststack" and \
 151                     row["tg_type"] in ("iperf", "vpp"):
 152                 if "BPS" not in tbs[dut][dver][infra]["ttype"]:
 153                     tbs[dut][dver][infra]["ttype"].append("BPS")
 154             elif row["test_type"] == "hoststack" and row["tg_type"] == "ab":
 155                 if "CPS" not in tbs[dut][dver][infra]["ttype"]:
 156                     tbs[dut][dver][infra]["ttype"].extend(("CPS", "RPS", ))
 157         self._tbs = tbs
 158
 159         # Read from files:
 160         self._html_layout = str()
 161         try:
 162             with open(self._html_layout_file, "r") as file_read:
 163                 self._html_layout = file_read.read()
 164         except IOError as err:
 165             raise RuntimeError(
 166                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 167             )
 168
 169         # Callbacks:
 170         if self._app is not None and hasattr(self, "callbacks"):
 171             self.callbacks(self._app)
 172
 173     @property
 174     def html_layout(self):
 175         return self._html_layout
 176
 177     def add_content(self):
 178         """Top level method which generated the web page.
 179
 180         It generates:
 181         - Store for user input data,
 182         - Navigation bar,
 183         - Main area with control panel and ploting area.
 184
 185         If no HTML layout is provided, an error message is displayed instead.
 186
 187         :returns: The HTML div with the whole page.
 188         :rtype: html.Div
 189         """
 190
 191         if self.html_layout and self._tbs:
 192             return html.Div(
 193                 id="div-main",
 194                 className="small",
 195                 children=[
 196                     dbc.Row(
 197                         id="row-navbar",
 198                         class_name="g-0",
 199                         children=[navbar_report((False, True, False, False)), ]
 200                     ),
 201                     dbc.Row(
 202                         id="row-main",
 203                         class_name="g-0",
 204                         children=[
 205                             dcc.Store(id="store-control-panel"),
 206                             dcc.Store(id="store-selected"),
 207                             dcc.Store(id="store-table-data"),
 208                             dcc.Store(id="store-filtered-table-data"),
 209                             dcc.Location(id="url", refresh=False),
 210                             self._add_ctrl_col(),
 211                             self._add_plotting_col()
 212                         ]
 213                     ),
 214                     dbc.Offcanvas(
 215                         class_name="w-75",
 216                         id="offcanvas-documentation",
 217                         title="Documentation",
 218                         placement="end",
 219                         is_open=False,
 220                         children=html.Iframe(
 221                             src=C.URL_DOC_REL_NOTES,
 222                             width="100%",
 223                             height="100%"
 224                         )
 225                     )
 226                 ]
 227             )
 228         else:
 229             return html.Div(
 230                 id="div-main-error",
 231                 children=[
 232                     dbc.Alert(
 233                         [
 234                             "An Error Occured"
 235                         ],
 236                         color="danger"
 237                     )
 238                 ]
 239             )
 240
 241     def _add_ctrl_col(self) -> dbc.Col:
 242         """Add column with controls. It is placed on the left side.
 243
 244         :returns: Column with the control panel.
 245         :rtype: dbc.Col
 246         """
 247         return dbc.Col([
 248             html.Div(
 249                 children=self._add_ctrl_panel(),
 250                 className="sticky-top"
 251             )
 252         ])
 253
 254     def _add_plotting_col(self) -> dbc.Col:
 255         """Add column with plots. It is placed on the right side.
 256
 257         :returns: Column with plots.
 258         :rtype: dbc.Col
 259         """
 260         return dbc.Col(
 261             id="col-plotting-area",
 262             children=[
 263                 dbc.Spinner(
 264                     children=[
 265                         dbc.Row(
 266                             id="plotting-area",
 267                             class_name="g-0 p-0",
 268                             children=[
 269                                 C.PLACEHOLDER
 270                             ]
 271                         )
 272                     ]
 273                 )
 274             ],
 275             width=9
 276         )
 277
 278     def _add_ctrl_panel(self) -> list:
 279         """Add control panel.
 280
 281         :returns: Control panel.
 282         :rtype: list
 283         """
 284
 285         reference = [
 286             dbc.Row(
 287                 class_name="g-0 p-1",
 288                 children=[
 289                     dbc.InputGroup(
 290                         [
 291                             dbc.InputGroupText("DUT"),
 292                             dbc.Select(
 293                                 id={"type": "ctrl-dd", "index": "dut"},
 294                                 placeholder="Select a Device under Test...",
 295                                 options=sorted(
 296                                     [
 297                                         {"label": k, "value": k} \
 298                                             for k in self._tbs.keys()
 299                                     ],
 300                                     key=lambda d: d["label"]
 301                                 )
 302                             )
 303                         ],
 304                         size="sm"
 305                     )
 306                 ]
 307             ),
 308             dbc.Row(
 309                 class_name="g-0 p-1",
 310                 children=[
 311                     dbc.InputGroup(
 312                         [
 313                             dbc.InputGroupText("CSIT and DUT Version"),
 314                             dbc.Select(
 315                                 id={"type": "ctrl-dd", "index": "dutver"},
 316                                 placeholder="Select a CSIT and DUT Version...")
 317                         ],
 318                         size="sm"
 319                     )
 320                 ]
 321             ),
 322             dbc.Row(
 323                 class_name="g-0 p-1",
 324                 children=[
 325                     dbc.InputGroup(
 326                         [
 327                             dbc.InputGroupText("Infra"),
 328                             dbc.Select(
 329                                 id={"type": "ctrl-dd", "index": "infra"},
 330                                 placeholder=\
 331                                     "Select a Physical Test Bed Topology..."
 332                             )
 333                         ],
 334                         size="sm"
 335                     )
 336                 ]
 337             ),
 338             dbc.Row(
 339                 class_name="g-0 p-1",
 340                 children=[
 341                     dbc.InputGroup(
 342                         [
 343                             dbc.InputGroupText("Frame Size"),
 344                             dbc.Checklist(
 345                                 id={"type": "ctrl-cl", "index": "frmsize"},
 346                                 inline=True,
 347                                 class_name="ms-2"
 348                             )
 349                         ],
 350                         style={"align-items": "center"},
 351                         size="sm"
 352                     )
 353                 ]
 354             ),
 355             dbc.Row(
 356                 class_name="g-0 p-1",
 357                 children=[
 358                     dbc.InputGroup(
 359                         [
 360                             dbc.InputGroupText("Number of Cores"),
 361                             dbc.Checklist(
 362                                 id={"type": "ctrl-cl", "index": "core"},
 363                                 inline=True,
 364                                 class_name="ms-2"
 365                             )
 366                         ],
 367                         style={"align-items": "center"},
 368                         size="sm"
 369                     )
 370                 ]
 371             ),
 372             dbc.Row(
 373                 class_name="g-0 p-1",
 374                 children=[
 375                     dbc.InputGroup(
 376                         [
 377                             dbc.InputGroupText("Measurement"),
 378                             dbc.Checklist(
 379                                 id={"type": "ctrl-cl", "index": "ttype"},
 380                                 inline=True,
 381                                 class_name="ms-2"
 382                             )
 383                         ],
 384                         style={"align-items": "center"},
 385                         size="sm"
 386                     )
 387                 ]
 388             )
 389         ]
 390
 391         compare = [
 392             dbc.Row(
 393                 class_name="g-0 p-1",
 394                 children=[
 395                     dbc.InputGroup(
 396                         [
 397                             dbc.InputGroupText("Parameter"),
 398                             dbc.Select(
 399                                 id={"type": "ctrl-dd", "index": "cmpprm"},
 400                                 placeholder="Select a Parameter..."
 401                             )
 402                         ],
 403                         size="sm"
 404                     )
 405                 ]
 406             ),
 407             dbc.Row(
 408                 class_name="g-0 p-1",
 409                 children=[
 410                     dbc.InputGroup(
 411                         [
 412                             dbc.InputGroupText("Value"),
 413                             dbc.Select(
 414                                 id={"type": "ctrl-dd", "index": "cmpval"},
 415                                 placeholder="Select a Value..."
 416                             )
 417                         ],
 418                         size="sm"
 419                     )
 420                 ]
 421             )
 422         ]
 423
 424         processing = [
 425             dbc.Row(
 426                 class_name="g-0 p-1",
 427                 children=[
 428                     dbc.InputGroup(
 429                         children = [
 430                             dbc.Checklist(
 431                                 id="normalize",
 432                                 options=[{
 433                                     "value": "normalize",
 434                                     "label": "Normalize to 2GHz CPU frequency"
 435                                 }],
 436                                 value=[],
 437                                 inline=True,
 438                                 class_name="ms-2"
 439                             ),
 440                             dbc.Checklist(
 441                                 id="outliers",
 442                                 options=[{
 443                                     "value": "outliers",
 444                                     "label": "Remove Extreme Outliers"
 445                                 }],
 446                                 value=[],
 447                                 inline=True,
 448                                 class_name="ms-2"
 449                             )
 450                         ],
 451                         style={"align-items": "center"},
 452                         size="sm"
 453                     )
 454                 ]
 455             )
 456         ]
 457
 458         return [
 459             dbc.Row(
 460                 dbc.Card(
 461                     [
 462                         dbc.CardHeader(
 463                             html.H5("Reference Value")
 464                         ),
 465                         dbc.CardBody(
 466                             children=reference,
 467                             class_name="g-0 p-0"
 468                         )
 469                     ],
 470                     color="secondary",
 471                     outline=True
 472                 ),
 473                 class_name="g-0 p-1"
 474             ),
 475             dbc.Row(
 476                 dbc.Card(
 477                     [
 478                         dbc.CardHeader(
 479                             html.H5("Compared Value")
 480                         ),
 481                         dbc.CardBody(
 482                             children=compare,
 483                             class_name="g-0 p-0"
 484                         )
 485                     ],
 486                     color="secondary",
 487                     outline=True
 488                 ),
 489                 class_name="g-0 p-1"
 490             ),
 491             dbc.Row(
 492                 dbc.Card(
 493                     [
 494                         dbc.CardHeader(
 495                             html.H5("Data Manipulations")
 496                         ),
 497                         dbc.CardBody(
 498                             children=processing,
 499                             class_name="g-0 p-0"
 500                         )
 501                     ],
 502                     color="secondary",
 503                     outline=True
 504                 ),
 505                 class_name="g-0 p-1"
 506             )
 507         ]
 508
 509     @staticmethod
 510     def _get_plotting_area(
 511             title: str,
 512             table: pd.DataFrame,
 513             url: str
 514         ) -> list:
 515         """Generate the plotting area with all its content.
 516
 517         :param title: The title of the comparison table.
 518         :param table: Comparison table to be displayed.
 519         :param url: URL to be displayed in the modal window.
 520         :type title: str
 521         :type table: pandas.DataFrame
 522         :type url: str
 523         :returns: List of rows with elements to be displayed in the plotting
 524             area.
 525         :rtype: list
 526         """
 527
 528         if table.empty:
 529             return dbc.Row(
 530                 dbc.Col(
 531                     children=dbc.Alert(
 532                         "No data for comparison.",
 533                         color="danger"
 534                     ),
 535                     class_name="g-0 p-1",
 536                 ),
 537                 class_name="g-0 p-0"
 538             )
 539
 540         cols = list()
 541         for idx, col in enumerate(table.columns):
 542             if idx == 0:
 543                 cols.append({
 544                     "name": ["", col],
 545                     "id": col,
 546                     "deletable": False,
 547                     "selectable": False,
 548                     "type": "text"
 549                 })
 550             else:
 551                 l_col = col.rsplit(" ", 2)
 552                 cols.append({
 553                     "name": [l_col[0], " ".join(l_col[-2:])],
 554                     "id": col,
 555                     "deletable": False,
 556                     "selectable": False,
 557                     "type": "numeric",
 558                     "format": Format(precision=2, scheme=Scheme.fixed)
 559                 })
 560
 561         return [
 562             dbc.Row(
 563                 children=html.H5(title),
 564                 class_name="g-0 p-1"
 565             ),
 566             dbc.Row(
 567                 children=[
 568                     dbc.Col(
 569                         children=dash_table.DataTable(
 570                             id={"type": "table", "index": "comparison"},
 571                             columns=cols,
 572                             data=table.to_dict("records"),
 573                             merge_duplicate_headers=True,
 574                             editable=False,
 575                             filter_action="custom",
 576                             filter_query="",
 577                             sort_action="native",
 578                             sort_mode="multi",
 579                             selected_columns=[],
 580                             selected_rows=[],
 581                             page_action="none",
 582                             style_cell={"textAlign": "right"},
 583                             style_cell_conditional=[{
 584                                 "if": {"column_id": "Test Name"},
 585                                 "textAlign": "left"
 586                             }]
 587                         ),
 588                         class_name="g-0 p-1"
 589                     )
 590                 ],
 591                 class_name="g-0 p-0"
 592             ),
 593             dbc.Row(
 594                 [
 595                     dbc.Col([html.Div(
 596                         [
 597                             dbc.Button(
 598                                 id="plot-btn-url",
 599                                 children="Show URL",
 600                                 class_name="me-1",
 601                                 color="info",
 602                                 style={
 603                                     "text-transform": "none",
 604                                     "padding": "0rem 1rem"
 605                                 }
 606                             ),
 607                             dbc.Modal(
 608                                 [
 609                                     dbc.ModalHeader(dbc.ModalTitle("URL")),
 610                                     dbc.ModalBody(url)
 611                                 ],
 612                                 id="plot-mod-url",
 613                                 size="xl",
 614                                 is_open=False,
 615                                 scrollable=True
 616                             ),
 617                             dbc.Button(
 618                                 id="plot-btn-download",
 619                                 children="Download Data",
 620                                 class_name="me-1",
 621                                 color="info",
 622                                 style={
 623                                     "text-transform": "none",
 624                                     "padding": "0rem 1rem"
 625                                 }
 626                             ),
 627                             dcc.Download(id="download-iterative-data")
 628                         ],
 629                         className=\
 630                             "d-grid gap-0 d-md-flex justify-content-md-end"
 631                     )])
 632                 ],
 633                 class_name="g-0 p-0"
 634             ),
 635             dbc.Row(
 636                 children=C.PLACEHOLDER,
 637                 class_name="g-0 p-1"
 638             )
 639         ]
 640
 641     def callbacks(self, app):
 642         """Callbacks for the whole application.
 643
 644         :param app: The application.
 645         :type app: Flask
 646         """
 647
 648         @app.callback(
 649             [
 650                 Output("store-control-panel", "data"),
 651                 Output("store-selected", "data"),
 652                 Output("store-table-data", "data"),
 653                 Output("store-filtered-table-data", "data"),
 654                 Output("plotting-area", "children"),
 655                 Output({"type": "table", "index": ALL}, "data"),
 656                 Output({"type": "ctrl-dd", "index": "dut"}, "value"),
 657                 Output({"type": "ctrl-dd", "index": "dutver"}, "options"),
 658                 Output({"type": "ctrl-dd", "index": "dutver"}, "disabled"),
 659                 Output({"type": "ctrl-dd", "index": "dutver"}, "value"),
 660                 Output({"type": "ctrl-dd", "index": "infra"}, "options"),
 661                 Output({"type": "ctrl-dd", "index": "infra"}, "disabled"),
 662                 Output({"type": "ctrl-dd", "index": "infra"}, "value"),
 663                 Output({"type": "ctrl-cl", "index": "core"}, "options"),
 664                 Output({"type": "ctrl-cl", "index": "core"}, "value"),
 665                 Output({"type": "ctrl-cl", "index": "frmsize"}, "options"),
 666                 Output({"type": "ctrl-cl", "index": "frmsize"}, "value"),
 667                 Output({"type": "ctrl-cl", "index": "ttype"}, "options"),
 668                 Output({"type": "ctrl-cl", "index": "ttype"}, "value"),
 669                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "options"),
 670                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "disabled"),
 671                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "value"),
 672                 Output({"type": "ctrl-dd", "index": "cmpval"}, "options"),
 673                 Output({"type": "ctrl-dd", "index": "cmpval"}, "disabled"),
 674                 Output({"type": "ctrl-dd", "index": "cmpval"}, "value"),
 675                 Output("normalize", "value"),
 676                 Output("outliers", "value")
 677             ],
 678             [
 679                 State("store-control-panel", "data"),
 680                 State("store-selected", "data"),
 681                 State("store-table-data", "data"),
 682                 State("store-filtered-table-data", "data"),
 683                 State({"type": "table", "index": ALL}, "data")
 684             ],
 685             [
 686                 Input("url", "href"),
 687                 Input("normalize", "value"),
 688                 Input("outliers", "value"),
 689                 Input({"type": "table", "index": ALL}, "filter_query"),
 690                 Input({"type": "ctrl-dd", "index": ALL}, "value"),
 691                 Input({"type": "ctrl-cl", "index": ALL}, "value"),
 692                 Input({"type": "ctrl-btn", "index": ALL}, "n_clicks")
 693             ]
 694         )
 695         def _update_application(
 696                 control_panel: dict,
 697                 selected: dict,
 698                 store_table_data: list,
 699                 filtered_data: list,
 700                 table_data: list,
 701                 href: str,
 702                 normalize: list,
 703                 outliers: bool,
 704                 table_filter: str,
 705                 *_
 706             ) -> tuple:
 707             """Update the application when the event is detected.
 708             """
 709
 710             ctrl_panel = ControlPanel(CP_PARAMS, control_panel)
 711
 712             if selected is None:
 713                 selected = {
 714                     "reference": {
 715                         "set": False,
 716                     },
 717                     "compare": {
 718                         "set": False,
 719                     }
 720                 }
 721
 722             # Parse the url:
 723             parsed_url = url_decode(href)
 724             if parsed_url:
 725                 url_params = parsed_url["params"]
 726             else:
 727                 url_params = None
 728
 729             on_draw = False
 730             plotting_area = no_update
 731
 732             trigger = Trigger(callback_context.triggered)
 733             if trigger.type == "url" and url_params:
 734                 process_url = False
 735                 try:
 736                     selected = literal_eval(url_params["selected"][0])
 737                     r_sel = selected["reference"]["selection"]
 738                     c_sel = selected["compare"]
 739                     normalize = literal_eval(url_params["norm"][0])
 740                     try:  # Necessary for backward compatibility
 741                         outliers = literal_eval(url_params["outliers"][0])
 742                     except (KeyError, IndexError, AttributeError):
 743                         outliers = list()
 744                     process_url = bool(
 745                         (selected["reference"]["set"] == True) and
 746                         (c_sel["set"] == True)
 747                     )
 748                 except (KeyError, IndexError, AttributeError):
 749                     pass
 750                 if process_url:
 751                     ctrl_panel.set({
 752                         "dut-val": r_sel["dut"],
 753                         "dutver-opt": generate_options(
 754                             self._tbs[r_sel["dut"]].keys()
 755                         ),
 756                         "dutver-dis": False,
 757                         "dutver-val": r_sel["dutver"],
 758                         "infra-opt": generate_options(
 759                             self._tbs[r_sel["dut"]][r_sel["dutver"]].keys()
 760                         ),
 761                         "infra-dis": False,
 762                         "infra-val": r_sel["infra"],
 763                         "core-opt": generate_options(
 764                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 765                                 [r_sel["infra"]]["core"]
 766                         ),
 767                         "core-val": r_sel["core"],
 768                         "frmsize-opt": generate_options(
 769                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 770                                 [r_sel["infra"]]["fsize"]
 771                         ),
 772                         "frmsize-val": r_sel["frmsize"],
 773                         "ttype-opt": generate_options(
 774                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 775                                 [r_sel["infra"]]["ttype"]
 776                         ),
 777                         "ttype-val": r_sel["ttype"],
 778                         "normalize-val": normalize,
 779                         "outliers-val": outliers
 780                     })
 781                     opts = list()
 782                     for itm, label in CMP_PARAMS.items():
 783                         if len(ctrl_panel.get(f"{itm}-opt")) > 1:
 784                             opts.append({"label": label, "value": itm})
 785                     ctrl_panel.set({
 786                         "cmp-par-opt": opts,
 787                         "cmp-par-dis": False,
 788                         "cmp-par-val": c_sel["parameter"]
 789                     })
 790                     opts = list()
 791                     for itm in ctrl_panel.get(f"{c_sel['parameter']}-opt"):
 792                         set_val = ctrl_panel.get(f"{c_sel['parameter']}-val")
 793                         if isinstance(set_val, list):
 794                             if itm["value"] not in set_val:
 795                                 opts.append(itm)
 796                         else:
 797                             if itm["value"] != set_val:
 798                                 opts.append(itm)
 799                     ctrl_panel.set({
 800                         "cmp-val-opt": opts,
 801                         "cmp-val-dis": False,
 802                         "cmp-val-val": c_sel["value"]
 803                     })
 804                     on_draw = True
 805             elif trigger.type == "normalize":
 806                 ctrl_panel.set({"normalize-val": normalize})
 807                 on_draw = True
 808             elif trigger.type == "outliers":
 809                 ctrl_panel.set({"outliers-val": outliers})
 810                 on_draw = True
 811             elif trigger.type == "ctrl-dd":
 812                 if trigger.idx == "dut":
 813                     try:
 814                         opts = generate_options(self._tbs[trigger.value].keys())
 815                         disabled = False
 816                     except KeyError:
 817                         opts = list()
 818                         disabled = True
 819                     ctrl_panel.set({
 820                         "dut-val": trigger.value,
 821                         "dutver-opt": opts,
 822                         "dutver-dis": disabled,
 823                         "dutver-val": str(),
 824                         "infra-opt": list(),
 825                         "infra-dis": True,
 826                         "infra-val": str(),
 827                         "core-opt": list(),
 828                         "core-val": list(),
 829                         "frmsize-opt": list(),
 830                         "frmsize-val": list(),
 831                         "ttype-opt": list(),
 832                         "ttype-val": list(),
 833                         "cmp-par-opt": list(),
 834                         "cmp-par-dis": True,
 835                         "cmp-par-val": str(),
 836                         "cmp-val-opt": list(),
 837                         "cmp-val-dis": True,
 838                         "cmp-val-val": str()
 839                     })
 840                 elif trigger.idx == "dutver":
 841                     try:
 842                         dut = ctrl_panel.get("dut-val")
 843                         dver = self._tbs[dut][trigger.value]
 844                         opts = generate_options(dver.keys())
 845                         disabled = False
 846                     except KeyError:
 847                         opts = list()
 848                         disabled = True
 849                     ctrl_panel.set({
 850                         "dutver-val": trigger.value,
 851                         "infra-opt": opts,
 852                         "infra-dis": disabled,
 853                         "infra-val": str(),
 854                         "core-opt": list(),
 855                         "core-val": list(),
 856                         "frmsize-opt": list(),
 857                         "frmsize-val": list(),
 858                         "ttype-opt": list(),
 859                         "ttype-val": list(),
 860                         "cmp-par-opt": list(),
 861                         "cmp-par-dis": True,
 862                         "cmp-par-val": str(),
 863                         "cmp-val-opt": list(),
 864                         "cmp-val-dis": True,
 865                         "cmp-val-val": str()
 866                     })
 867                 elif trigger.idx == "infra":
 868                     dut = ctrl_panel.get("dut-val")
 869                     dver = ctrl_panel.get("dutver-val")
 870                     if all((dut, dver, trigger.value, )):
 871                         driver = self._tbs[dut][dver][trigger.value]
 872                         ctrl_panel.set({
 873                             "infra-val": trigger.value,
 874                             "core-opt": generate_options(driver["core"]),
 875                             "core-val": list(),
 876                             "frmsize-opt": generate_options(driver["fsize"]),
 877                             "frmsize-val": list(),
 878                             "ttype-opt": generate_options(driver["ttype"]),
 879                             "ttype-val": list(),
 880                             "cmp-par-opt": list(),
 881                             "cmp-par-dis": True,
 882                             "cmp-par-val": str(),
 883                             "cmp-val-opt": list(),
 884                             "cmp-val-dis": True,
 885                             "cmp-val-val": str()
 886                         })
 887                 elif trigger.idx == "cmpprm":
 888                     value = trigger.value
 889                     opts = list()
 890                     for itm in ctrl_panel.get(f"{value}-opt"):
 891                         set_val = ctrl_panel.get(f"{value}-val")
 892                         if isinstance(set_val, list):
 893                             if itm["value"] == "Latency":
 894                                 continue
 895                             if itm["value"] not in set_val:
 896                                 opts.append(itm)
 897                         else:
 898                             if itm["value"] != set_val:
 899                                 opts.append(itm)
 900                     ctrl_panel.set({
 901                         "cmp-par-val": value,
 902                         "cmp-val-opt": opts,
 903                         "cmp-val-dis": False,
 904                         "cmp-val-val": str()
 905                     })
 906                 elif trigger.idx == "cmpval":
 907                     ctrl_panel.set({"cmp-val-val": trigger.value})
 908                     selected["reference"] = {
 909                         "set": True,
 910                         "selection": {
 911                             "dut": ctrl_panel.get("dut-val"),
 912                             "dutver": ctrl_panel.get("dutver-val"),
 913                             "infra": ctrl_panel.get("infra-val"),
 914                             "core": ctrl_panel.get("core-val"),
 915                             "frmsize": ctrl_panel.get("frmsize-val"),
 916                             "ttype": ctrl_panel.get("ttype-val")
 917                         }
 918                     }
 919                     selected["compare"] = {
 920                         "set": True,
 921                         "parameter": ctrl_panel.get("cmp-par-val"),
 922                         "value": trigger.value
 923                     }
 924                     on_draw = True
 925             elif trigger.type == "ctrl-cl":
 926                 ctrl_panel.set({f"{trigger.idx}-val": trigger.value})
 927                 if all((ctrl_panel.get("core-val"),
 928                         ctrl_panel.get("frmsize-val"),
 929                         ctrl_panel.get("ttype-val"), )):
 930                     if "Latency" in ctrl_panel.get("ttype-val"):
 931                         ctrl_panel.set({"ttype-val": ["Latency", ]})
 932                     opts = list()
 933                     for itm, label in CMP_PARAMS.items():
 934                         if "Latency" in ctrl_panel.get("ttype-val") and \
 935                                 itm == "ttype":
 936                             continue
 937                         if len(ctrl_panel.get(f"{itm}-opt")) > 1:
 938                             if isinstance(ctrl_panel.get(f"{itm}-val"), list):
 939                                 if len(ctrl_panel.get(f"{itm}-opt")) == \
 940                                         len(ctrl_panel.get(f"{itm}-val")):
 941                                     continue
 942                             opts.append({"label": label, "value": itm})
 943                     ctrl_panel.set({
 944                         "cmp-par-opt": opts,
 945                         "cmp-par-dis": False,
 946                         "cmp-par-val": str(),
 947                         "cmp-val-opt": list(),
 948                         "cmp-val-dis": True,
 949                         "cmp-val-val": str()
 950                     })
 951                 else:
 952                     ctrl_panel.set({
 953                         "cmp-par-opt": list(),
 954                         "cmp-par-dis": True,
 955                         "cmp-par-val": str(),
 956                         "cmp-val-opt": list(),
 957                         "cmp-val-dis": True,
 958                         "cmp-val-val": str()
 959                     })
 960             elif trigger.type == "table" and trigger.idx == "comparison":
 961                 filtered_data = filter_table_data(
 962                     store_table_data,
 963                     table_filter[0]
 964                 )
 965                 table_data = [filtered_data, ]
 966
 967             if all((on_draw, selected["reference"]["set"],
 968                     selected["compare"]["set"], )):
 969                 title, table = comparison_table(
 970                     data=self._data,
 971                     selected=selected,
 972                     normalize=normalize,
 973                     format="html",
 974                     remove_outliers=outliers
 975                 )
 976                 plotting_area = self._get_plotting_area(
 977                     title=title,
 978                     table=table,
 979                     url=gen_new_url(
 980                         parsed_url,
 981                         params={
 982                             "selected": selected,
 983                             "norm": normalize,
 984                             "outliers": outliers
 985                         }
 986                     )
 987                 )
 988                 store_table_data = table.to_dict("records")
 989                 filtered_data = store_table_data
 990                 if table_data:
 991                     table_data = [store_table_data, ]
 992
 993             ret_val = [
 994                 ctrl_panel.panel,
 995                 selected,
 996                 store_table_data,
 997                 filtered_data,
 998                 plotting_area,
 999                 table_data
1000             ]
1001             ret_val.extend(ctrl_panel.values)
1002             return ret_val
1003
1004         @app.callback(
1005             Output("plot-mod-url", "is_open"),
1006             Input("plot-btn-url", "n_clicks"),
1007             State("plot-mod-url", "is_open")
1008         )
1009         def toggle_plot_mod_url(n, is_open):
1010             """Toggle the modal window with url.
1011             """
1012             if n:
1013                 return not is_open
1014             return is_open
1015
1016         @app.callback(
1017             Output("download-iterative-data", "data"),
1018             State("store-table-data", "data"),
1019             State("store-filtered-table-data", "data"),
1020             Input("plot-btn-download", "n_clicks"),
1021             prevent_initial_call=True
1022         )
1023         def _download_comparison_data(
1024                 table_data: list,
1025                 filtered_table_data: list,
1026                 _: int
1027             ) -> dict:
1028             """Download the data.
1029
1030             :param table_data: Original unfiltered table data.
1031             :param filtered_table_data: Filtered table data.
1032             :type table_data: list
1033             :type filtered_table_data: list
1034             :returns: dict of data frame content (base64 encoded) and meta data
1035                 used by the Download component.
1036             :rtype: dict
1037             """
1038
1039             if not table_data:
1040                 raise PreventUpdate
1041
1042             if filtered_table_data:
1043                 table = pd.DataFrame.from_records(filtered_table_data)
1044             else:
1045                 table = pd.DataFrame.from_records(table_data)
1046
1047             return dcc.send_data_frame(table.to_csv, C.COMP_DOWNLOAD_FILE_NAME)
1048
1049         @app.callback(
1050             Output("offcanvas-documentation", "is_open"),
1051             Input("btn-documentation", "n_clicks"),
1052             State("offcanvas-documentation", "is_open")
1053         )
1054         def toggle_offcanvas_documentation(n_clicks, is_open):
1055             if n_clicks:
1056                 return not is_open
1057             return is_open