csit.infra.dash/app/cdash/comparisons/layout.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import pandas as pd
  18 import dash_bootstrap_components as dbc
  19
  20 from flask import Flask
  21 from dash import dcc, html, dash_table, callback_context, no_update, ALL
  22 from dash import Input, Output, State
  23 from dash.exceptions import PreventUpdate
  24 from dash.dash_table.Format import Format, Scheme
  25 from ast import literal_eval
  26
  27 from ..utils.constants import Constants as C
  28 from ..utils.control_panel import ControlPanel
  29 from ..utils.trigger import Trigger
  30 from ..utils.url_processing import url_decode
  31 from ..utils.utils import generate_options, gen_new_url
  32 from .tables import comparison_table, filter_table_data
  33
  34
  35 # Control panel partameters and their default values.
  36 CP_PARAMS = {
  37     "dut-val": str(),
  38     "dutver-opt": list(),
  39     "dutver-dis": True,
  40     "dutver-val": str(),
  41     "infra-opt": list(),
  42     "infra-dis": True,
  43     "infra-val": str(),
  44     "core-opt": list(),
  45     "core-val": list(),
  46     "frmsize-opt": list(),
  47     "frmsize-val": list(),
  48     "ttype-opt": list(),
  49     "ttype-val": list(),
  50     "cmp-par-opt": list(),
  51     "cmp-par-dis": True,
  52     "cmp-par-val": str(),
  53     "cmp-val-opt": list(),
  54     "cmp-val-dis": True,
  55     "cmp-val-val": str(),
  56     "normalize-val": list()
  57 }
  58
  59 # List of comparable parameters.
  60 CMP_PARAMS = {
  61     "dutver": "Release and Version",
  62     "infra": "Infrastructure",
  63     "frmsize": "Frame Size",
  64     "core": "Number of Cores",
  65     "ttype": "Measurement"
  66 }
  67
  68
  69 class Layout:
  70     """The layout of the dash app and the callbacks.
  71     """
  72
  73     def __init__(
  74             self,
  75             app: Flask,
  76             data_iterative: pd.DataFrame,
  77             html_layout_file: str
  78         ) -> None:
  79         """Initialization:
  80         - save the input parameters,
  81         - prepare data for the control panel,
  82         - read HTML layout file,
  83
  84         :param app: Flask application running the dash application.
  85         :param data_iterative: Iterative data to be used in comparison tables.
  86         :param html_layout_file: Path and name of the file specifying the HTML
  87             layout of the dash application.
  88         :type app: Flask
  89         :type data_iterative: pandas.DataFrame
  90         :type html_layout_file: str
  91         """
  92
  93         # Inputs
  94         self._app = app
  95         self._html_layout_file = html_layout_file
  96         self._data = data_iterative
  97
  98         # Get structure of tests:
  99         tbs = dict()
 100         cols = [
 101             "job", "test_id", "test_type", "dut_type", "dut_version", "tg_type",
 102             "release", "passed"
 103         ]
 104         for _, row in self._data[cols].drop_duplicates().iterrows():
 105             lst_job = row["job"].split("-")
 106             dut = lst_job[1]
 107             dver = f"{row['release']}-{row['dut_version']}"
 108             tbed = "-".join(lst_job[-2:])
 109             lst_test_id = row["test_id"].split(".")
 110
 111             suite = lst_test_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
 112                 replace("2n-", "")
 113             test = lst_test_id[-1]
 114             nic = suite.split("-")[0]
 115             for driver in C.DRIVERS:
 116                 if driver in test:
 117                     drv = driver.replace("-", "_")
 118                     test = test.replace(f"{driver}-", "")
 119                     break
 120             else:
 121                 drv = "dpdk"
 122             infra = "-".join((tbed, nic, drv))
 123             lst_test = test.split("-")
 124             fsize = lst_test[0]
 125             core = lst_test[1] if lst_test[1] else "8C"
 126
 127             if tbs.get(dut, None) is None:
 128                 tbs[dut] = dict()
 129             if tbs[dut].get(dver, None) is None:
 130                 tbs[dut][dver] = dict()
 131             if tbs[dut][dver].get(infra, None) is None:
 132                 tbs[dut][dver][infra] = dict()
 133                 tbs[dut][dver][infra]["core"] = list()
 134                 tbs[dut][dver][infra]["fsize"] = list()
 135                 tbs[dut][dver][infra]["ttype"] = list()
 136             if core.upper() not in tbs[dut][dver][infra]["core"]:
 137                 tbs[dut][dver][infra]["core"].append(core.upper())
 138             if fsize.upper() not in tbs[dut][dver][infra]["fsize"]:
 139                 tbs[dut][dver][infra]["fsize"].append(fsize.upper())
 140             if row["test_type"] == "mrr":
 141                 if "MRR" not in tbs[dut][dver][infra]["ttype"]:
 142                     tbs[dut][dver][infra]["ttype"].append("MRR")
 143             elif row["test_type"] == "ndrpdr":
 144                 if "NDR" not in tbs[dut][dver][infra]["ttype"]:
 145                     tbs[dut][dver][infra]["ttype"].extend(
 146                         ("NDR", "PDR", "Latency")
 147                     )
 148             elif row["test_type"] == "hoststack" and \
 149                     row["tg_type"] in ("iperf", "vpp"):
 150                 if "BPS" not in tbs[dut][dver][infra]["ttype"]:
 151                     tbs[dut][dver][infra]["ttype"].append("BPS")
 152             elif row["test_type"] == "hoststack" and row["tg_type"] == "ab":
 153                 if "CPS" not in tbs[dut][dver][infra]["ttype"]:
 154                     tbs[dut][dver][infra]["ttype"].extend(("CPS", "RPS", ))
 155         self._tbs = tbs
 156
 157         # Read from files:
 158         self._html_layout = str()
 159         try:
 160             with open(self._html_layout_file, "r") as file_read:
 161                 self._html_layout = file_read.read()
 162         except IOError as err:
 163             raise RuntimeError(
 164                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 165             )
 166
 167         # Callbacks:
 168         if self._app is not None and hasattr(self, "callbacks"):
 169             self.callbacks(self._app)
 170
 171     @property
 172     def html_layout(self):
 173         return self._html_layout
 174
 175     def add_content(self):
 176         """Top level method which generated the web page.
 177
 178         It generates:
 179         - Store for user input data,
 180         - Navigation bar,
 181         - Main area with control panel and ploting area.
 182
 183         If no HTML layout is provided, an error message is displayed instead.
 184
 185         :returns: The HTML div with the whole page.
 186         :rtype: html.Div
 187         """
 188
 189         if self.html_layout and self._tbs:
 190             return html.Div(
 191                 id="div-main",
 192                 className="small",
 193                 children=[
 194                     dbc.Row(
 195                         id="row-navbar",
 196                         class_name="g-0",
 197                         children=[
 198                             self._add_navbar()
 199                         ]
 200                     ),
 201                     dbc.Row(
 202                         id="row-main",
 203                         class_name="g-0",
 204                         children=[
 205                             dcc.Store(id="store-control-panel"),
 206                             dcc.Store(id="store-selected"),
 207                             dcc.Store(id="store-table-data"),
 208                             dcc.Store(id="store-filtered-table-data"),
 209                             dcc.Location(id="url", refresh=False),
 210                             self._add_ctrl_col(),
 211                             self._add_plotting_col()
 212                         ]
 213                     ),
 214                     dbc.Offcanvas(
 215                         class_name="w-75",
 216                         id="offcanvas-documentation",
 217                         title="Documentation",
 218                         placement="end",
 219                         is_open=False,
 220                         children=html.Iframe(
 221                             src=C.URL_DOC_REL_NOTES,
 222                             width="100%",
 223                             height="100%"
 224                         )
 225                     )
 226                 ]
 227             )
 228         else:
 229             return html.Div(
 230                 id="div-main-error",
 231                 children=[
 232                     dbc.Alert(
 233                         [
 234                             "An Error Occured"
 235                         ],
 236                         color="danger"
 237                     )
 238                 ]
 239             )
 240
 241     def _add_navbar(self):
 242         """Add nav element with navigation panel. It is placed on the top.
 243
 244         :returns: Navigation bar.
 245         :rtype: dbc.NavbarSimple
 246         """
 247         return dbc.NavbarSimple(
 248             id="navbarsimple-main",
 249             children=[
 250                 dbc.NavItem(dbc.NavLink(
 251                     C.REPORT_TITLE,
 252                     external_link=True,
 253                     href="/report"
 254                 )),
 255                 dbc.NavItem(dbc.NavLink(
 256                     "Comparisons",
 257                     active=True,
 258                     external_link=True,
 259                     href="/comparisons"
 260                 )),
 261                 dbc.NavItem(dbc.NavLink(
 262                     "Coverage Data",
 263                     external_link=True,
 264                     href="/coverage"
 265                 )),
 266                 dbc.NavItem(dbc.NavLink(
 267                     "Documentation",
 268                     id="btn-documentation",
 269                 ))
 270             ],
 271             brand=C.BRAND,
 272             brand_href="/",
 273             brand_external_link=True,
 274             class_name="p-2",
 275             fluid=True
 276         )
 277
 278     def _add_ctrl_col(self) -> dbc.Col:
 279         """Add column with controls. It is placed on the left side.
 280
 281         :returns: Column with the control panel.
 282         :rtype: dbc.Col
 283         """
 284         return dbc.Col([
 285             html.Div(
 286                 children=self._add_ctrl_panel(),
 287                 className="sticky-top"
 288             )
 289         ])
 290
 291     def _add_plotting_col(self) -> dbc.Col:
 292         """Add column with plots. It is placed on the right side.
 293
 294         :returns: Column with plots.
 295         :rtype: dbc.Col
 296         """
 297         return dbc.Col(
 298             id="col-plotting-area",
 299             children=[
 300                 dbc.Spinner(
 301                     children=[
 302                         dbc.Row(
 303                             id="plotting-area",
 304                             class_name="g-0 p-0",
 305                             children=[
 306                                 C.PLACEHOLDER
 307                             ]
 308                         )
 309                     ]
 310                 )
 311             ],
 312             width=9
 313         )
 314
 315     def _add_ctrl_panel(self) -> list:
 316         """Add control panel.
 317
 318         :returns: Control panel.
 319         :rtype: list
 320         """
 321
 322         reference = [
 323             dbc.Row(
 324                 class_name="g-0 p-1",
 325                 children=[
 326                     dbc.InputGroup(
 327                         [
 328                             dbc.InputGroupText("DUT"),
 329                             dbc.Select(
 330                                 id={"type": "ctrl-dd", "index": "dut"},
 331                                 placeholder="Select a Device under Test...",
 332                                 options=sorted(
 333                                     [
 334                                         {"label": k, "value": k} \
 335                                             for k in self._tbs.keys()
 336                                     ],
 337                                     key=lambda d: d["label"]
 338                                 )
 339                             )
 340                         ],
 341                         size="sm"
 342                     )
 343                 ]
 344             ),
 345             dbc.Row(
 346                 class_name="g-0 p-1",
 347                 children=[
 348                     dbc.InputGroup(
 349                         [
 350                             dbc.InputGroupText("CSIT and DUT Version"),
 351                             dbc.Select(
 352                                 id={"type": "ctrl-dd", "index": "dutver"},
 353                                 placeholder="Select a CSIT and DUT Version...")
 354                         ],
 355                         size="sm"
 356                     )
 357                 ]
 358             ),
 359             dbc.Row(
 360                 class_name="g-0 p-1",
 361                 children=[
 362                     dbc.InputGroup(
 363                         [
 364                             dbc.InputGroupText("Infra"),
 365                             dbc.Select(
 366                                 id={"type": "ctrl-dd", "index": "infra"},
 367                                 placeholder=\
 368                                     "Select a Physical Test Bed Topology..."
 369                             )
 370                         ],
 371                         size="sm"
 372                     )
 373                 ]
 374             ),
 375             dbc.Row(
 376                 class_name="g-0 p-1",
 377                 children=[
 378                     dbc.InputGroup(
 379                         [
 380                             dbc.InputGroupText("Frame Size"),
 381                             dbc.Checklist(
 382                                 id={"type": "ctrl-cl", "index": "frmsize"},
 383                                 inline=True,
 384                                 class_name="ms-2"
 385                             )
 386                         ],
 387                         style={"align-items": "center"},
 388                         size="sm"
 389                     )
 390                 ]
 391             ),
 392             dbc.Row(
 393                 class_name="g-0 p-1",
 394                 children=[
 395                     dbc.InputGroup(
 396                         [
 397                             dbc.InputGroupText("Number of Cores"),
 398                             dbc.Checklist(
 399                                 id={"type": "ctrl-cl", "index": "core"},
 400                                 inline=True,
 401                                 class_name="ms-2"
 402                             )
 403                         ],
 404                         style={"align-items": "center"},
 405                         size="sm"
 406                     )
 407                 ]
 408             ),
 409             dbc.Row(
 410                 class_name="g-0 p-1",
 411                 children=[
 412                     dbc.InputGroup(
 413                         [
 414                             dbc.InputGroupText("Measurement"),
 415                             dbc.Checklist(
 416                                 id={"type": "ctrl-cl", "index": "ttype"},
 417                                 inline=True,
 418                                 class_name="ms-2"
 419                             )
 420                         ],
 421                         style={"align-items": "center"},
 422                         size="sm"
 423                     )
 424                 ]
 425             )
 426         ]
 427
 428         compare = [
 429             dbc.Row(
 430                 class_name="g-0 p-1",
 431                 children=[
 432                     dbc.InputGroup(
 433                         [
 434                             dbc.InputGroupText("Parameter"),
 435                             dbc.Select(
 436                                 id={"type": "ctrl-dd", "index": "cmpprm"},
 437                                 placeholder="Select a Parameter..."
 438                             )
 439                         ],
 440                         size="sm"
 441                     )
 442                 ]
 443             ),
 444             dbc.Row(
 445                 class_name="g-0 p-1",
 446                 children=[
 447                     dbc.InputGroup(
 448                         [
 449                             dbc.InputGroupText("Value"),
 450                             dbc.Select(
 451                                 id={"type": "ctrl-dd", "index": "cmpval"},
 452                                 placeholder="Select a Value..."
 453                             )
 454                         ],
 455                         size="sm"
 456                     )
 457                 ]
 458             )
 459         ]
 460
 461         normalize = [
 462             dbc.Row(
 463                 class_name="g-0 p-1",
 464                 children=[
 465                     dbc.InputGroup(
 466                         dbc.Checklist(
 467                             id="normalize",
 468                             options=[{
 469                                 "value": "normalize",
 470                                 "label": "Normalize to 2GHz CPU frequency"
 471                             }],
 472                             value=[],
 473                             inline=True,
 474                             class_name="ms-2"
 475                         ),
 476                         style={"align-items": "center"},
 477                         size="sm"
 478                     )
 479                 ]
 480             )
 481         ]
 482
 483         return [
 484             dbc.Row(
 485                 dbc.Card(
 486                     [
 487                         dbc.CardHeader(
 488                             html.H5("Reference Value")
 489                         ),
 490                         dbc.CardBody(
 491                             children=reference,
 492                             class_name="g-0 p-0"
 493                         )
 494                     ],
 495                     color="secondary",
 496                     outline=True
 497                 ),
 498                 class_name="g-0 p-1"
 499             ),
 500             dbc.Row(
 501                 dbc.Card(
 502                     [
 503                         dbc.CardHeader(
 504                             html.H5("Compared Value")
 505                         ),
 506                         dbc.CardBody(
 507                             children=compare,
 508                             class_name="g-0 p-0"
 509                         )
 510                     ],
 511                     color="secondary",
 512                     outline=True
 513                 ),
 514                 class_name="g-0 p-1"
 515             ),
 516             dbc.Row(
 517                 dbc.Card(
 518                     [
 519                         dbc.CardHeader(
 520                             html.H5("Normalization")
 521                         ),
 522                         dbc.CardBody(
 523                             children=normalize,
 524                             class_name="g-0 p-0"
 525                         )
 526                     ],
 527                     color="secondary",
 528                     outline=True
 529                 ),
 530                 class_name="g-0 p-1"
 531             )
 532         ]
 533
 534     @staticmethod
 535     def _get_plotting_area(
 536             title: str,
 537             table: pd.DataFrame,
 538             url: str
 539         ) -> list:
 540         """Generate the plotting area with all its content.
 541
 542         :param title: The title of the comparison table..
 543         :param table: Comparison table to be displayed.
 544         :param url: URL to be displayed in the modal window.
 545         :type title: str
 546         :type table: pandas.DataFrame
 547         :type url: str
 548         :returns: List of rows with elements to be displayed in the plotting
 549             area.
 550         :rtype: list
 551         """
 552
 553         if table.empty:
 554             return dbc.Row(
 555                 dbc.Col(
 556                     children=dbc.Alert(
 557                         "No data for comparison.",
 558                         color="danger"
 559                     ),
 560                     class_name="g-0 p-1",
 561                 ),
 562                 class_name="g-0 p-0"
 563             )
 564
 565         cols = list()
 566         for idx, col in enumerate(table.columns):
 567             if idx == 0:
 568                 cols.append({
 569                     "name": ["", col],
 570                     "id": col,
 571                     "deletable": False,
 572                     "selectable": False,
 573                     "type": "text"
 574                 })
 575             else:
 576                 l_col = col.rsplit(" ", 2)
 577                 cols.append({
 578                     "name": [l_col[0], " ".join(l_col[-2:])],
 579                     "id": col,
 580                     "deletable": False,
 581                     "selectable": False,
 582                     "type": "numeric",
 583                     "format": Format(precision=2, scheme=Scheme.fixed)
 584                 })
 585
 586         return [
 587             dbc.Row(
 588                 children=html.H5(title),
 589                 class_name="g-0 p-1"
 590             ),
 591             dbc.Row(
 592                 children=[
 593                     dbc.Col(
 594                         children=dash_table.DataTable(
 595                             id={"type": "table", "index": "comparison"},
 596                             columns=cols,
 597                             data=table.to_dict("records"),
 598                             merge_duplicate_headers=True,
 599                             editable=False,
 600                             filter_action="custom",
 601                             filter_query="",
 602                             sort_action="native",
 603                             sort_mode="multi",
 604                             selected_columns=[],
 605                             selected_rows=[],
 606                             page_action="none",
 607                             style_cell={"textAlign": "right"},
 608                             style_cell_conditional=[{
 609                                 "if": {"column_id": "Test Name"},
 610                                 "textAlign": "left"
 611                             }]
 612                         ),
 613                         class_name="g-0 p-1"
 614                     )
 615                 ],
 616                 class_name="g-0 p-0"
 617             ),
 618             dbc.Row(
 619                 [
 620                     dbc.Col([html.Div(
 621                         [
 622                             dbc.Button(
 623                                 id="plot-btn-url",
 624                                 children="Show URL",
 625                                 class_name="me-1",
 626                                 color="info",
 627                                 style={
 628                                     "text-transform": "none",
 629                                     "padding": "0rem 1rem"
 630                                 }
 631                             ),
 632                             dbc.Modal(
 633                                 [
 634                                     dbc.ModalHeader(dbc.ModalTitle("URL")),
 635                                     dbc.ModalBody(url)
 636                                 ],
 637                                 id="plot-mod-url",
 638                                 size="xl",
 639                                 is_open=False,
 640                                 scrollable=True
 641                             ),
 642                             dbc.Button(
 643                                 id="plot-btn-download",
 644                                 children="Download Data",
 645                                 class_name="me-1",
 646                                 color="info",
 647                                 style={
 648                                     "text-transform": "none",
 649                                     "padding": "0rem 1rem"
 650                                 }
 651                             ),
 652                             dcc.Download(id="download-iterative-data")
 653                         ],
 654                         className=\
 655                             "d-grid gap-0 d-md-flex justify-content-md-end"
 656                     )])
 657                 ],
 658                 class_name="g-0 p-0"
 659             ),
 660             dbc.Row(
 661                 children=C.PLACEHOLDER,
 662                 class_name="g-0 p-1"
 663             )
 664         ]
 665
 666     def callbacks(self, app):
 667         """Callbacks for the whole application.
 668
 669         :param app: The application.
 670         :type app: Flask
 671         """
 672
 673         @app.callback(
 674             [
 675                 Output("store-control-panel", "data"),
 676                 Output("store-selected", "data"),
 677                 Output("store-table-data", "data"),
 678                 Output("store-filtered-table-data", "data"),
 679                 Output("plotting-area", "children"),
 680                 Output({"type": "table", "index": ALL}, "data"),
 681                 Output({"type": "ctrl-dd", "index": "dut"}, "value"),
 682                 Output({"type": "ctrl-dd", "index": "dutver"}, "options"),
 683                 Output({"type": "ctrl-dd", "index": "dutver"}, "disabled"),
 684                 Output({"type": "ctrl-dd", "index": "dutver"}, "value"),
 685                 Output({"type": "ctrl-dd", "index": "infra"}, "options"),
 686                 Output({"type": "ctrl-dd", "index": "infra"}, "disabled"),
 687                 Output({"type": "ctrl-dd", "index": "infra"}, "value"),
 688                 Output({"type": "ctrl-cl", "index": "core"}, "options"),
 689                 Output({"type": "ctrl-cl", "index": "core"}, "value"),
 690                 Output({"type": "ctrl-cl", "index": "frmsize"}, "options"),
 691                 Output({"type": "ctrl-cl", "index": "frmsize"}, "value"),
 692                 Output({"type": "ctrl-cl", "index": "ttype"}, "options"),
 693                 Output({"type": "ctrl-cl", "index": "ttype"}, "value"),
 694                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "options"),
 695                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "disabled"),
 696                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "value"),
 697                 Output({"type": "ctrl-dd", "index": "cmpval"}, "options"),
 698                 Output({"type": "ctrl-dd", "index": "cmpval"}, "disabled"),
 699                 Output({"type": "ctrl-dd", "index": "cmpval"}, "value"),
 700                 Output("normalize", "value")
 701             ],
 702             [
 703                 State("store-control-panel", "data"),
 704                 State("store-selected", "data"),
 705                 State("store-table-data", "data"),
 706                 State("store-filtered-table-data", "data"),
 707                 State({"type": "table", "index": ALL}, "data")
 708             ],
 709             [
 710                 Input("url", "href"),
 711                 Input("normalize", "value"),
 712                 Input({"type": "table", "index": ALL}, "filter_query"),
 713                 Input({"type": "ctrl-dd", "index": ALL}, "value"),
 714                 Input({"type": "ctrl-cl", "index": ALL}, "value"),
 715                 Input({"type": "ctrl-btn", "index": ALL}, "n_clicks")
 716             ]
 717         )
 718         def _update_application(
 719                 control_panel: dict,
 720                 selected: dict,
 721                 store_table_data: list,
 722                 filtered_data: list,
 723                 table_data: list,
 724                 href: str,
 725                 normalize: list,
 726                 table_filter: str,
 727                 *_
 728             ) -> tuple:
 729             """Update the application when the event is detected.
 730             """
 731
 732             ctrl_panel = ControlPanel(CP_PARAMS, control_panel)
 733
 734             if selected is None:
 735                 selected = {
 736                     "reference": {
 737                         "set": False,
 738                     },
 739                     "compare": {
 740                         "set": False,
 741                     }
 742                 }
 743
 744             # Parse the url:
 745             parsed_url = url_decode(href)
 746             if parsed_url:
 747                 url_params = parsed_url["params"]
 748             else:
 749                 url_params = None
 750
 751             on_draw = False
 752             plotting_area = no_update
 753
 754             trigger = Trigger(callback_context.triggered)
 755             if trigger.type == "url" and url_params:
 756                 process_url = False
 757                 try:
 758                     selected = literal_eval(url_params["selected"][0])
 759                     r_sel = selected["reference"]["selection"]
 760                     c_sel = selected["compare"]
 761                     normalize = literal_eval(url_params["norm"][0])
 762                     process_url = bool(
 763                         (selected["reference"]["set"] == True) and
 764                         (c_sel["set"] == True)
 765                     )
 766                 except (KeyError, IndexError, AttributeError):
 767                     pass
 768                 if process_url:
 769                     ctrl_panel.set({
 770                         "dut-val": r_sel["dut"],
 771                         "dutver-opt": generate_options(
 772                             self._tbs[r_sel["dut"]].keys()
 773                         ),
 774                         "dutver-dis": False,
 775                         "dutver-val": r_sel["dutver"],
 776                         "infra-opt": generate_options(
 777                             self._tbs[r_sel["dut"]][r_sel["dutver"]].keys()
 778                         ),
 779                         "infra-dis": False,
 780                         "infra-val": r_sel["infra"],
 781                         "core-opt": generate_options(
 782                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 783                                 [r_sel["infra"]]["core"]
 784                         ),
 785                         "core-val": r_sel["core"],
 786                         "frmsize-opt": generate_options(
 787                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 788                                 [r_sel["infra"]]["fsize"]
 789                         ),
 790                         "frmsize-val": r_sel["frmsize"],
 791                         "ttype-opt": generate_options(
 792                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 793                                 [r_sel["infra"]]["ttype"]
 794                         ),
 795                         "ttype-val": r_sel["ttype"],
 796                         "normalize-val": normalize
 797                     })
 798                     opts = list()
 799                     for itm, label in CMP_PARAMS.items():
 800                         if len(ctrl_panel.get(f"{itm}-opt")) > 1:
 801                             opts.append({"label": label, "value": itm})
 802                     ctrl_panel.set({
 803                         "cmp-par-opt": opts,
 804                         "cmp-par-dis": False,
 805                         "cmp-par-val": c_sel["parameter"]
 806                     })
 807                     opts = list()
 808                     for itm in ctrl_panel.get(f"{c_sel['parameter']}-opt"):
 809                         set_val = ctrl_panel.get(f"{c_sel['parameter']}-val")
 810                         if isinstance(set_val, list):
 811                             if itm["value"] not in set_val:
 812                                 opts.append(itm)
 813                         else:
 814                             if itm["value"] != set_val:
 815                                 opts.append(itm)
 816                     ctrl_panel.set({
 817                         "cmp-val-opt": opts,
 818                         "cmp-val-dis": False,
 819                         "cmp-val-val": c_sel["value"]
 820                     })
 821                     on_draw = True
 822             elif trigger.type == "normalize":
 823                 ctrl_panel.set({"normalize-val": normalize})
 824                 on_draw = True
 825             elif trigger.type == "ctrl-dd":
 826                 if trigger.idx == "dut":
 827                     try:
 828                         opts = generate_options(self._tbs[trigger.value].keys())
 829                         disabled = False
 830                     except KeyError:
 831                         opts = list()
 832                         disabled = True
 833                     ctrl_panel.set({
 834                         "dut-val": trigger.value,
 835                         "dutver-opt": opts,
 836                         "dutver-dis": disabled,
 837                         "dutver-val": str(),
 838                         "infra-opt": list(),
 839                         "infra-dis": True,
 840                         "infra-val": str(),
 841                         "core-opt": list(),
 842                         "core-val": list(),
 843                         "frmsize-opt": list(),
 844                         "frmsize-val": list(),
 845                         "ttype-opt": list(),
 846                         "ttype-val": list(),
 847                         "cmp-par-opt": list(),
 848                         "cmp-par-dis": True,
 849                         "cmp-par-val": str(),
 850                         "cmp-val-opt": list(),
 851                         "cmp-val-dis": True,
 852                         "cmp-val-val": str()
 853                     })
 854                 elif trigger.idx == "dutver":
 855                     try:
 856                         dut = ctrl_panel.get("dut-val")
 857                         dver = self._tbs[dut][trigger.value]
 858                         opts = generate_options(dver.keys())
 859                         disabled = False
 860                     except KeyError:
 861                         opts = list()
 862                         disabled = True
 863                     ctrl_panel.set({
 864                         "dutver-val": trigger.value,
 865                         "infra-opt": opts,
 866                         "infra-dis": disabled,
 867                         "infra-val": str(),
 868                         "core-opt": list(),
 869                         "core-val": list(),
 870                         "frmsize-opt": list(),
 871                         "frmsize-val": list(),
 872                         "ttype-opt": list(),
 873                         "ttype-val": list(),
 874                         "cmp-par-opt": list(),
 875                         "cmp-par-dis": True,
 876                         "cmp-par-val": str(),
 877                         "cmp-val-opt": list(),
 878                         "cmp-val-dis": True,
 879                         "cmp-val-val": str()
 880                     })
 881                 elif trigger.idx == "infra":
 882                     dut = ctrl_panel.get("dut-val")
 883                     dver = ctrl_panel.get("dutver-val")
 884                     if all((dut, dver, trigger.value, )):
 885                         driver = self._tbs[dut][dver][trigger.value]
 886                         ctrl_panel.set({
 887                             "infra-val": trigger.value,
 888                             "core-opt": generate_options(driver["core"]),
 889                             "core-val": list(),
 890                             "frmsize-opt": generate_options(driver["fsize"]),
 891                             "frmsize-val": list(),
 892                             "ttype-opt": generate_options(driver["ttype"]),
 893                             "ttype-val": list(),
 894                             "cmp-par-opt": list(),
 895                             "cmp-par-dis": True,
 896                             "cmp-par-val": str(),
 897                             "cmp-val-opt": list(),
 898                             "cmp-val-dis": True,
 899                             "cmp-val-val": str()
 900                         })
 901                 elif trigger.idx == "cmpprm":
 902                     value = trigger.value
 903                     opts = list()
 904                     for itm in ctrl_panel.get(f"{value}-opt"):
 905                         set_val = ctrl_panel.get(f"{value}-val")
 906                         if isinstance(set_val, list):
 907                             if itm["value"] == "Latency":
 908                                 continue
 909                             if itm["value"] not in set_val:
 910                                 opts.append(itm)
 911                         else:
 912                             if itm["value"] != set_val:
 913                                 opts.append(itm)
 914                     ctrl_panel.set({
 915                         "cmp-par-val": value,
 916                         "cmp-val-opt": opts,
 917                         "cmp-val-dis": False,
 918                         "cmp-val-val": str()
 919                     })
 920                 elif trigger.idx == "cmpval":
 921                     ctrl_panel.set({"cmp-val-val": trigger.value})
 922                     selected["reference"] = {
 923                         "set": True,
 924                         "selection": {
 925                             "dut": ctrl_panel.get("dut-val"),
 926                             "dutver": ctrl_panel.get("dutver-val"),
 927                             "infra": ctrl_panel.get("infra-val"),
 928                             "core": ctrl_panel.get("core-val"),
 929                             "frmsize": ctrl_panel.get("frmsize-val"),
 930                             "ttype": ctrl_panel.get("ttype-val")
 931                         }
 932                     }
 933                     selected["compare"] = {
 934                         "set": True,
 935                         "parameter": ctrl_panel.get("cmp-par-val"),
 936                         "value": trigger.value
 937                     }
 938                     on_draw = True
 939             elif trigger.type == "ctrl-cl":
 940                 ctrl_panel.set({f"{trigger.idx}-val": trigger.value})
 941                 if all((ctrl_panel.get("core-val"),
 942                         ctrl_panel.get("frmsize-val"),
 943                         ctrl_panel.get("ttype-val"), )):
 944                     if "Latency" in ctrl_panel.get("ttype-val"):
 945                         ctrl_panel.set({"ttype-val": ["Latency", ]})
 946                     opts = list()
 947                     for itm, label in CMP_PARAMS.items():
 948                         if "Latency" in ctrl_panel.get("ttype-val") and \
 949                                 itm == "ttype":
 950                             continue
 951                         if len(ctrl_panel.get(f"{itm}-opt")) > 1:
 952                             if isinstance(ctrl_panel.get(f"{itm}-val"), list):
 953                                 if len(ctrl_panel.get(f"{itm}-opt")) == \
 954                                         len(ctrl_panel.get(f"{itm}-val")):
 955                                     continue
 956                             opts.append({"label": label, "value": itm})
 957                     ctrl_panel.set({
 958                         "cmp-par-opt": opts,
 959                         "cmp-par-dis": False,
 960                         "cmp-par-val": str(),
 961                         "cmp-val-opt": list(),
 962                         "cmp-val-dis": True,
 963                         "cmp-val-val": str()
 964                     })
 965                 else:
 966                     ctrl_panel.set({
 967                         "cmp-par-opt": list(),
 968                         "cmp-par-dis": True,
 969                         "cmp-par-val": str(),
 970                         "cmp-val-opt": list(),
 971                         "cmp-val-dis": True,
 972                         "cmp-val-val": str()
 973                     })
 974             elif trigger.type == "table" and trigger.idx == "comparison":
 975                 filtered_data = filter_table_data(
 976                     store_table_data,
 977                     table_filter[0]
 978                 )
 979                 table_data = [filtered_data, ]
 980
 981             if all((on_draw, selected["reference"]["set"],
 982                     selected["compare"]["set"], )):
 983                 title, table = comparison_table(self._data, selected, normalize)
 984                 plotting_area = self._get_plotting_area(
 985                     title=title,
 986                     table=table,
 987                     url=gen_new_url(
 988                         parsed_url,
 989                         params={"selected": selected, "norm": normalize}
 990                     )
 991                 )
 992                 store_table_data = table.to_dict("records")
 993                 filtered_data = store_table_data
 994                 if table_data:
 995                     table_data = [store_table_data, ]
 996
 997             ret_val = [
 998                 ctrl_panel.panel,
 999                 selected,
1000                 store_table_data,
1001                 filtered_data,
1002                 plotting_area,
1003                 table_data
1004             ]
1005             ret_val.extend(ctrl_panel.values)
1006             return ret_val
1007
1008         @app.callback(
1009             Output("plot-mod-url", "is_open"),
1010             Input("plot-btn-url", "n_clicks"),
1011             State("plot-mod-url", "is_open")
1012         )
1013         def toggle_plot_mod_url(n, is_open):
1014             """Toggle the modal window with url.
1015             """
1016             if n:
1017                 return not is_open
1018             return is_open
1019
1020         @app.callback(
1021             Output("download-iterative-data", "data"),
1022             State("store-table-data", "data"),
1023             State("store-filtered-table-data", "data"),
1024             Input("plot-btn-download", "n_clicks"),
1025             prevent_initial_call=True
1026         )
1027         def _download_comparison_data(
1028                 table_data: list,
1029                 filtered_table_data: list,
1030                 _: int
1031             ) -> dict:
1032             """Download the data.
1033
1034             :param table_data: Original unfiltered table data.
1035             :param filtered_table_data: Filtered table data.
1036             :type table_data: list
1037             :type filtered_table_data: list
1038             :returns: dict of data frame content (base64 encoded) and meta data
1039                 used by the Download component.
1040             :rtype: dict
1041             """
1042
1043             if not table_data:
1044                 raise PreventUpdate
1045
1046             if filtered_table_data:
1047                 table = pd.DataFrame.from_records(filtered_table_data)
1048             else:
1049                 table = pd.DataFrame.from_records(table_data)
1050
1051             return dcc.send_data_frame(table.to_csv, C.COMP_DOWNLOAD_FILE_NAME)
1052
1053         @app.callback(
1054             Output("offcanvas-documentation", "is_open"),
1055             Input("btn-documentation", "n_clicks"),
1056             State("offcanvas-documentation", "is_open")
1057         )
1058         def toggle_offcanvas_documentation(n_clicks, is_open):
1059             if n_clicks:
1060                 return not is_open
1061             return is_open