csit.infra.dash/app/cdash/comparisons/layout.py

   1 # Copyright (c) 2024 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import pandas as pd
  18 import dash_bootstrap_components as dbc
  19
  20 from flask import Flask
  21 from dash import dcc, html, dash_table, callback_context, no_update, ALL
  22 from dash import Input, Output, State
  23 from dash.exceptions import PreventUpdate
  24 from dash.dash_table.Format import Format, Scheme
  25 from ast import literal_eval
  26
  27 from ..utils.constants import Constants as C
  28 from ..utils.control_panel import ControlPanel
  29 from ..utils.trigger import Trigger
  30 from ..utils.url_processing import url_decode
  31 from ..utils.utils import generate_options, gen_new_url, navbar_report, \
  32     filter_table_data
  33 from .tables import comparison_table
  34
  35
  36 # Control panel partameters and their default values.
  37 CP_PARAMS = {
  38     "dut-val": str(),
  39     "dutver-opt": list(),
  40     "dutver-dis": True,
  41     "dutver-val": str(),
  42     "infra-opt": list(),
  43     "infra-dis": True,
  44     "infra-val": str(),
  45     "core-opt": list(),
  46     "core-val": list(),
  47     "frmsize-opt": list(),
  48     "frmsize-val": list(),
  49     "ttype-opt": list(),
  50     "ttype-val": list(),
  51     "cmp-par-opt": list(),
  52     "cmp-par-dis": True,
  53     "cmp-par-val": str(),
  54     "cmp-val-opt": list(),
  55     "cmp-val-dis": True,
  56     "cmp-val-val": str(),
  57     "normalize-val": list(),
  58     "outliers-val": list()
  59 }
  60
  61 # List of comparable parameters.
  62 CMP_PARAMS = {
  63     "dutver": "Release and Version",
  64     "infra": "Infrastructure",
  65     "frmsize": "Frame Size",
  66     "core": "Number of Cores",
  67     "ttype": "Measurement"
  68 }
  69
  70
  71 class Layout:
  72     """The layout of the dash app and the callbacks.
  73     """
  74
  75     def __init__(
  76             self,
  77             app: Flask,
  78             data_iterative: pd.DataFrame,
  79             html_layout_file: str
  80         ) -> None:
  81         """Initialization:
  82         - save the input parameters,
  83         - prepare data for the control panel,
  84         - read HTML layout file,
  85
  86         :param app: Flask application running the dash application.
  87         :param data_iterative: Iterative data to be used in comparison tables.
  88         :param html_layout_file: Path and name of the file specifying the HTML
  89             layout of the dash application.
  90         :type app: Flask
  91         :type data_iterative: pandas.DataFrame
  92         :type html_layout_file: str
  93         """
  94
  95         # Inputs
  96         self._app = app
  97         self._html_layout_file = html_layout_file
  98         self._data = data_iterative
  99
 100         # Get structure of tests:
 101         tbs = dict()
 102         cols = [
 103             "job", "test_id", "test_type", "dut_type", "dut_version", "tg_type",
 104             "release", "passed"
 105         ]
 106         for _, row in self._data[cols].drop_duplicates().iterrows():
 107             lst_job = row["job"].split("-")
 108             dut = lst_job[1]
 109             dver = f"{row['release']}-{row['dut_version']}"
 110             tbed = "-".join(lst_job[-2:])
 111             lst_test_id = row["test_id"].split(".")
 112
 113             suite = lst_test_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
 114                 replace("2n-", "")
 115             test = lst_test_id[-1]
 116             nic = suite.split("-")[0]
 117             for driver in C.DRIVERS:
 118                 if driver in test:
 119                     drv = driver.replace("-", "_")
 120                     test = test.replace(f"{driver}-", "")
 121                     break
 122             else:
 123                 drv = "dpdk"
 124             infra = "-".join((tbed, nic, drv))
 125             lst_test = test.split("-")
 126             fsize = lst_test[0]
 127             core = lst_test[1] if lst_test[1] else "8C"
 128
 129             if tbs.get(dut, None) is None:
 130                 tbs[dut] = dict()
 131             if tbs[dut].get(dver, None) is None:
 132                 tbs[dut][dver] = dict()
 133             if tbs[dut][dver].get(infra, None) is None:
 134                 tbs[dut][dver][infra] = dict()
 135                 tbs[dut][dver][infra]["core"] = list()
 136                 tbs[dut][dver][infra]["fsize"] = list()
 137                 tbs[dut][dver][infra]["ttype"] = list()
 138             if core.upper() not in tbs[dut][dver][infra]["core"]:
 139                 tbs[dut][dver][infra]["core"].append(core.upper())
 140             if fsize.upper() not in tbs[dut][dver][infra]["fsize"]:
 141                 tbs[dut][dver][infra]["fsize"].append(fsize.upper())
 142             if row["test_type"] == "mrr":
 143                 if "MRR" not in tbs[dut][dver][infra]["ttype"]:
 144                     tbs[dut][dver][infra]["ttype"].append("MRR")
 145             elif row["test_type"] == "ndrpdr":
 146                 if "NDR" not in tbs[dut][dver][infra]["ttype"]:
 147                     tbs[dut][dver][infra]["ttype"].extend(
 148                         ("NDR", "PDR", "Latency")
 149                     )
 150             elif row["test_type"] == "hoststack" and \
 151                     row["tg_type"] in ("iperf", "vpp"):
 152                 if "BPS" not in tbs[dut][dver][infra]["ttype"]:
 153                     tbs[dut][dver][infra]["ttype"].append("BPS")
 154             elif row["test_type"] == "hoststack" and row["tg_type"] == "ab":
 155                 if "CPS" not in tbs[dut][dver][infra]["ttype"]:
 156                     tbs[dut][dver][infra]["ttype"].extend(("CPS", "RPS", ))
 157         self._tbs = tbs
 158
 159         # Read from files:
 160         self._html_layout = str()
 161         try:
 162             with open(self._html_layout_file, "r") as file_read:
 163                 self._html_layout = file_read.read()
 164         except IOError as err:
 165             raise RuntimeError(
 166                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 167             )
 168
 169         # Callbacks:
 170         if self._app is not None and hasattr(self, "callbacks"):
 171             self.callbacks(self._app)
 172
 173     @property
 174     def html_layout(self):
 175         return self._html_layout
 176
 177     def add_content(self):
 178         """Top level method which generated the web page.
 179
 180         It generates:
 181         - Store for user input data,
 182         - Navigation bar,
 183         - Main area with control panel and ploting area.
 184
 185         If no HTML layout is provided, an error message is displayed instead.
 186
 187         :returns: The HTML div with the whole page.
 188         :rtype: html.Div
 189         """
 190
 191         if self.html_layout and self._tbs:
 192             return html.Div(
 193                 id="div-main",
 194                 className="small",
 195                 children=[
 196                     dbc.Row(
 197                         id="row-navbar",
 198                         class_name="g-0",
 199                         children=[navbar_report((False, True, False, False)), ]
 200                     ),
 201                     dbc.Row(
 202                         id="row-main",
 203                         class_name="g-0",
 204                         children=[
 205                             dcc.Store(id="store-control-panel"),
 206                             dcc.Store(id="store-selected"),
 207                             dcc.Store(id="store-table-data"),
 208                             dcc.Store(id="store-filtered-table-data"),
 209                             dcc.Location(id="url", refresh=False),
 210                             self._add_ctrl_col(),
 211                             self._add_plotting_col()
 212                         ]
 213                     ),
 214                     dbc.Offcanvas(
 215                         class_name="w-75",
 216                         id="offcanvas-documentation",
 217                         title="Documentation",
 218                         placement="end",
 219                         is_open=False,
 220                         children=html.Iframe(
 221                             src=C.URL_DOC_REL_NOTES,
 222                             width="100%",
 223                             height="100%"
 224                         )
 225                     )
 226                 ]
 227             )
 228         else:
 229             return html.Div(
 230                 id="div-main-error",
 231                 children=[
 232                     dbc.Alert(
 233                         [
 234                             "An Error Occured"
 235                         ],
 236                         color="danger"
 237                     )
 238                 ]
 239             )
 240
 241     def _add_ctrl_col(self) -> dbc.Col:
 242         """Add column with controls. It is placed on the left side.
 243
 244         :returns: Column with the control panel.
 245         :rtype: dbc.Col
 246         """
 247         return dbc.Col([
 248             html.Div(
 249                 children=self._add_ctrl_panel(),
 250                 className="sticky-top"
 251             )
 252         ])
 253
 254     def _add_plotting_col(self) -> dbc.Col:
 255         """Add column with plots. It is placed on the right side.
 256
 257         :returns: Column with plots.
 258         :rtype: dbc.Col
 259         """
 260         return dbc.Col(
 261             id="col-plotting-area",
 262             children=[
 263                 dbc.Spinner(
 264                     children=[
 265                         dbc.Row(
 266                             id="plotting-area",
 267                             class_name="g-0 p-0",
 268                             children=[
 269                                 C.PLACEHOLDER
 270                             ]
 271                         )
 272                     ]
 273                 )
 274             ],
 275             width=9
 276         )
 277
 278     def _add_ctrl_panel(self) -> list:
 279         """Add control panel.
 280
 281         :returns: Control panel.
 282         :rtype: list
 283         """
 284
 285         reference = [
 286             dbc.Row(
 287                 class_name="g-0 p-1",
 288                 children=[
 289                     dbc.InputGroup(
 290                         [
 291                             dbc.InputGroupText("DUT"),
 292                             dbc.Select(
 293                                 id={"type": "ctrl-dd", "index": "dut"},
 294                                 placeholder="Select a Device under Test...",
 295                                 options=sorted(
 296                                     [
 297                                         {"label": k, "value": k} \
 298                                             for k in self._tbs.keys()
 299                                     ],
 300                                     key=lambda d: d["label"]
 301                                 )
 302                             )
 303                         ],
 304                         size="sm"
 305                     )
 306                 ]
 307             ),
 308             dbc.Row(
 309                 class_name="g-0 p-1",
 310                 children=[
 311                     dbc.InputGroup(
 312                         [
 313                             dbc.InputGroupText("CSIT and DUT Version"),
 314                             dbc.Select(
 315                                 id={"type": "ctrl-dd", "index": "dutver"},
 316                                 placeholder="Select a CSIT and DUT Version...")
 317                         ],
 318                         size="sm"
 319                     )
 320                 ]
 321             ),
 322             dbc.Row(
 323                 class_name="g-0 p-1",
 324                 children=[
 325                     dbc.InputGroup(
 326                         [
 327                             dbc.InputGroupText("Infra"),
 328                             dbc.Select(
 329                                 id={"type": "ctrl-dd", "index": "infra"},
 330                                 placeholder=\
 331                                     "Select a Physical Test Bed Topology..."
 332                             )
 333                         ],
 334                         size="sm"
 335                     )
 336                 ]
 337             ),
 338             dbc.Row(
 339                 class_name="g-0 p-1",
 340                 children=[
 341                     dbc.InputGroup(
 342                         [
 343                             dbc.InputGroupText("Frame Size"),
 344                             dbc.Checklist(
 345                                 id={"type": "ctrl-cl", "index": "frmsize"},
 346                                 inline=True,
 347                                 class_name="ms-2"
 348                             )
 349                         ],
 350                         style={"align-items": "center"},
 351                         size="sm"
 352                     )
 353                 ]
 354             ),
 355             dbc.Row(
 356                 class_name="g-0 p-1",
 357                 children=[
 358                     dbc.InputGroup(
 359                         [
 360                             dbc.InputGroupText("Number of Cores"),
 361                             dbc.Checklist(
 362                                 id={"type": "ctrl-cl", "index": "core"},
 363                                 inline=True,
 364                                 class_name="ms-2"
 365                             )
 366                         ],
 367                         style={"align-items": "center"},
 368                         size="sm"
 369                     )
 370                 ]
 371             ),
 372             dbc.Row(
 373                 class_name="g-0 p-1",
 374                 children=[
 375                     dbc.InputGroup(
 376                         [
 377                             dbc.InputGroupText("Measurement"),
 378                             dbc.Checklist(
 379                                 id={"type": "ctrl-cl", "index": "ttype"},
 380                                 inline=True,
 381                                 class_name="ms-2"
 382                             )
 383                         ],
 384                         style={"align-items": "center"},
 385                         size="sm"
 386                     )
 387                 ]
 388             )
 389         ]
 390
 391         compare = [
 392             dbc.Row(
 393                 class_name="g-0 p-1",
 394                 children=[
 395                     dbc.InputGroup(
 396                         [
 397                             dbc.InputGroupText("Parameter"),
 398                             dbc.Select(
 399                                 id={"type": "ctrl-dd", "index": "cmpprm"},
 400                                 placeholder="Select a Parameter..."
 401                             )
 402                         ],
 403                         size="sm"
 404                     )
 405                 ]
 406             ),
 407             dbc.Row(
 408                 class_name="g-0 p-1",
 409                 children=[
 410                     dbc.InputGroup(
 411                         [
 412                             dbc.InputGroupText("Value"),
 413                             dbc.Select(
 414                                 id={"type": "ctrl-dd", "index": "cmpval"},
 415                                 placeholder="Select a Value..."
 416                             )
 417                         ],
 418                         size="sm"
 419                     )
 420                 ]
 421             )
 422         ]
 423
 424         processing = [
 425             dbc.Row(
 426                 class_name="g-0 p-1",
 427                 children=[
 428                     dbc.InputGroup(
 429                         children = [
 430                             dbc.Checklist(
 431                                 id="normalize",
 432                                 options=[{
 433                                     "value": "normalize",
 434                                     "label": "Normalize to 2GHz CPU frequency"
 435                                 }],
 436                                 value=[],
 437                                 inline=True,
 438                                 class_name="ms-2"
 439                             ),
 440                             dbc.Checklist(
 441                                 id="outliers",
 442                                 options=[{
 443                                     "value": "outliers",
 444                                     "label": "Remove Extreme Outliers"
 445                                 }],
 446                                 value=[],
 447                                 inline=True,
 448                                 class_name="ms-2"
 449                             )
 450                         ],
 451                         style={"align-items": "center"},
 452                         size="sm"
 453                     )
 454                 ]
 455             )
 456         ]
 457
 458         return [
 459             dbc.Row(
 460                 dbc.Card(
 461                     [
 462                         dbc.CardHeader(
 463                             html.H5("Reference Value")
 464                         ),
 465                         dbc.CardBody(
 466                             children=reference,
 467                             class_name="g-0 p-0"
 468                         )
 469                     ],
 470                     color="secondary",
 471                     outline=True
 472                 ),
 473                 class_name="g-0 p-1"
 474             ),
 475             dbc.Row(
 476                 dbc.Card(
 477                     [
 478                         dbc.CardHeader(
 479                             html.H5("Compared Value")
 480                         ),
 481                         dbc.CardBody(
 482                             children=compare,
 483                             class_name="g-0 p-0"
 484                         )
 485                     ],
 486                     color="secondary",
 487                     outline=True
 488                 ),
 489                 class_name="g-0 p-1"
 490             ),
 491             dbc.Row(
 492                 dbc.Card(
 493                     [
 494                         dbc.CardHeader(
 495                             html.H5("Data Manipulations")
 496                         ),
 497                         dbc.CardBody(
 498                             children=processing,
 499                             class_name="g-0 p-0"
 500                         )
 501                     ],
 502                     color="secondary",
 503                     outline=True
 504                 ),
 505                 class_name="g-0 p-1"
 506             )
 507         ]
 508
 509     @staticmethod
 510     def _get_plotting_area(
 511             title: str,
 512             table: pd.DataFrame,
 513             url: str
 514         ) -> list:
 515         """Generate the plotting area with all its content.
 516
 517         :param title: The title of the comparison table.
 518         :param table: Comparison table to be displayed.
 519         :param url: URL to be displayed in the modal window.
 520         :type title: str
 521         :type table: pandas.DataFrame
 522         :type url: str
 523         :returns: List of rows with elements to be displayed in the plotting
 524             area.
 525         :rtype: list
 526         """
 527
 528         if table.empty:
 529             return dbc.Row(
 530                 dbc.Col(
 531                     children=dbc.Alert(
 532                         "No data for comparison.",
 533                         color="danger"
 534                     ),
 535                     class_name="g-0 p-1",
 536                 ),
 537                 class_name="g-0 p-0"
 538             )
 539
 540         cols = list()
 541         for idx, col in enumerate(table.columns):
 542             if idx == 0:
 543                 cols.append({
 544                     "name": ["", col],
 545                     "id": col,
 546                     "deletable": False,
 547                     "selectable": False,
 548                     "type": "text"
 549                 })
 550             else:
 551                 l_col = col.rsplit(" ", 2)
 552                 cols.append({
 553                     "name": [l_col[0], " ".join(l_col[-2:])],
 554                     "id": col,
 555                     "deletable": False,
 556                     "selectable": False,
 557                     "type": "numeric",
 558                     "format": Format(precision=2, scheme=Scheme.fixed)
 559                 })
 560
 561         return [
 562             dbc.Row(
 563                 children=html.H5(title),
 564                 class_name="g-0 p-1"
 565             ),
 566             dbc.Row(
 567                 children=[
 568                     dbc.Col(
 569                         children=dash_table.DataTable(
 570                             id={"type": "table", "index": "comparison"},
 571                             columns=cols,
 572                             data=table.to_dict("records"),
 573                             merge_duplicate_headers=True,
 574                             editable=False,
 575                             filter_action="custom",
 576                             filter_query="",
 577                             sort_action="native",
 578                             sort_mode="multi",
 579                             selected_columns=[],
 580                             selected_rows=[],
 581                             page_action="none",
 582                             style_cell={"textAlign": "right"},
 583                             style_cell_conditional=[{
 584                                 "if": {"column_id": "Test Name"},
 585                                 "textAlign": "left"
 586                             }]
 587                         ),
 588                         class_name="g-0 p-1"
 589                     )
 590                 ],
 591                 class_name="g-0 p-0"
 592             ),
 593             dbc.Row(
 594                 [
 595                     dbc.Col([html.Div(
 596                         [
 597                             dbc.Button(
 598                                 id="plot-btn-url",
 599                                 children="Show URL",
 600                                 class_name="me-1",
 601                                 color="info",
 602                                 style={
 603                                     "text-transform": "none",
 604                                     "padding": "0rem 1rem"
 605                                 }
 606                             ),
 607                             dbc.Modal(
 608                                 [
 609                                     dbc.ModalHeader(dbc.ModalTitle("URL")),
 610                                     dbc.ModalBody(url)
 611                                 ],
 612                                 id="plot-mod-url",
 613                                 size="xl",
 614                                 is_open=False,
 615                                 scrollable=True
 616                             ),
 617                             dbc.Button(
 618                                 id="plot-btn-download",
 619                                 children="Download Table",
 620                                 class_name="me-1",
 621                                 color="info",
 622                                 style={
 623                                     "text-transform": "none",
 624                                     "padding": "0rem 1rem"
 625                                 }
 626                             ),
 627                             dcc.Download(id="download-iterative-data"),
 628                             dbc.Button(
 629                                 id="plot-btn-download-raw",
 630                                 children="Download Raw Data",
 631                                 class_name="me-1",
 632                                 color="info",
 633                                 style={
 634                                     "text-transform": "none",
 635                                     "padding": "0rem 1rem"
 636                                 }
 637                             ),
 638                             dcc.Download(id="download-raw-data")
 639                         ],
 640                         className=\
 641                             "d-grid gap-0 d-md-flex justify-content-md-end"
 642                     )])
 643                 ],
 644                 class_name="g-0 p-0"
 645             ),
 646             dbc.Row(
 647                 children=C.PLACEHOLDER,
 648                 class_name="g-0 p-1"
 649             )
 650         ]
 651
 652     def callbacks(self, app):
 653         """Callbacks for the whole application.
 654
 655         :param app: The application.
 656         :type app: Flask
 657         """
 658
 659         @app.callback(
 660             [
 661                 Output("store-control-panel", "data"),
 662                 Output("store-selected", "data"),
 663                 Output("store-table-data", "data"),
 664                 Output("store-filtered-table-data", "data"),
 665                 Output("plotting-area", "children"),
 666                 Output({"type": "table", "index": ALL}, "data"),
 667                 Output({"type": "ctrl-dd", "index": "dut"}, "value"),
 668                 Output({"type": "ctrl-dd", "index": "dutver"}, "options"),
 669                 Output({"type": "ctrl-dd", "index": "dutver"}, "disabled"),
 670                 Output({"type": "ctrl-dd", "index": "dutver"}, "value"),
 671                 Output({"type": "ctrl-dd", "index": "infra"}, "options"),
 672                 Output({"type": "ctrl-dd", "index": "infra"}, "disabled"),
 673                 Output({"type": "ctrl-dd", "index": "infra"}, "value"),
 674                 Output({"type": "ctrl-cl", "index": "core"}, "options"),
 675                 Output({"type": "ctrl-cl", "index": "core"}, "value"),
 676                 Output({"type": "ctrl-cl", "index": "frmsize"}, "options"),
 677                 Output({"type": "ctrl-cl", "index": "frmsize"}, "value"),
 678                 Output({"type": "ctrl-cl", "index": "ttype"}, "options"),
 679                 Output({"type": "ctrl-cl", "index": "ttype"}, "value"),
 680                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "options"),
 681                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "disabled"),
 682                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "value"),
 683                 Output({"type": "ctrl-dd", "index": "cmpval"}, "options"),
 684                 Output({"type": "ctrl-dd", "index": "cmpval"}, "disabled"),
 685                 Output({"type": "ctrl-dd", "index": "cmpval"}, "value"),
 686                 Output("normalize", "value"),
 687                 Output("outliers", "value")
 688             ],
 689             [
 690                 State("store-control-panel", "data"),
 691                 State("store-selected", "data"),
 692                 State("store-table-data", "data"),
 693                 State("store-filtered-table-data", "data"),
 694                 State({"type": "table", "index": ALL}, "data")
 695             ],
 696             [
 697                 Input("url", "href"),
 698                 Input("normalize", "value"),
 699                 Input("outliers", "value"),
 700                 Input({"type": "table", "index": ALL}, "filter_query"),
 701                 Input({"type": "ctrl-dd", "index": ALL}, "value"),
 702                 Input({"type": "ctrl-cl", "index": ALL}, "value"),
 703                 Input({"type": "ctrl-btn", "index": ALL}, "n_clicks")
 704             ]
 705         )
 706         def _update_application(
 707                 control_panel: dict,
 708                 selected: dict,
 709                 store_table_data: list,
 710                 filtered_data: list,
 711                 table_data: list,
 712                 href: str,
 713                 normalize: list,
 714                 outliers: bool,
 715                 table_filter: str,
 716                 *_
 717             ) -> tuple:
 718             """Update the application when the event is detected.
 719             """
 720
 721             ctrl_panel = ControlPanel(CP_PARAMS, control_panel)
 722
 723             if selected is None:
 724                 selected = {
 725                     "reference": {
 726                         "set": False,
 727                     },
 728                     "compare": {
 729                         "set": False,
 730                     }
 731                 }
 732
 733             # Parse the url:
 734             parsed_url = url_decode(href)
 735             if parsed_url:
 736                 url_params = parsed_url["params"]
 737             else:
 738                 url_params = None
 739
 740             on_draw = False
 741             plotting_area = no_update
 742
 743             trigger = Trigger(callback_context.triggered)
 744             if trigger.type == "url" and url_params:
 745                 process_url = False
 746                 try:
 747                     selected = literal_eval(url_params["selected"][0])
 748                     r_sel = selected["reference"]["selection"]
 749                     c_sel = selected["compare"]
 750                     normalize = literal_eval(url_params["norm"][0])
 751                     try:  # Necessary for backward compatibility
 752                         outliers = literal_eval(url_params["outliers"][0])
 753                     except (KeyError, IndexError, AttributeError):
 754                         outliers = list()
 755                     process_url = bool(
 756                         (selected["reference"]["set"] == True) and
 757                         (c_sel["set"] == True)
 758                     )
 759                 except (KeyError, IndexError, AttributeError):
 760                     pass
 761                 if process_url:
 762                     ctrl_panel.set({
 763                         "dut-val": r_sel["dut"],
 764                         "dutver-opt": generate_options(
 765                             self._tbs[r_sel["dut"]].keys()
 766                         ),
 767                         "dutver-dis": False,
 768                         "dutver-val": r_sel["dutver"],
 769                         "infra-opt": generate_options(
 770                             self._tbs[r_sel["dut"]][r_sel["dutver"]].keys()
 771                         ),
 772                         "infra-dis": False,
 773                         "infra-val": r_sel["infra"],
 774                         "core-opt": generate_options(
 775                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 776                                 [r_sel["infra"]]["core"]
 777                         ),
 778                         "core-val": r_sel["core"],
 779                         "frmsize-opt": generate_options(
 780                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 781                                 [r_sel["infra"]]["fsize"]
 782                         ),
 783                         "frmsize-val": r_sel["frmsize"],
 784                         "ttype-opt": generate_options(
 785                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 786                                 [r_sel["infra"]]["ttype"]
 787                         ),
 788                         "ttype-val": r_sel["ttype"],
 789                         "normalize-val": normalize,
 790                         "outliers-val": outliers
 791                     })
 792                     opts = list()
 793                     for itm, label in CMP_PARAMS.items():
 794                         if len(ctrl_panel.get(f"{itm}-opt")) > 1:
 795                             opts.append({"label": label, "value": itm})
 796                     ctrl_panel.set({
 797                         "cmp-par-opt": opts,
 798                         "cmp-par-dis": False,
 799                         "cmp-par-val": c_sel["parameter"]
 800                     })
 801                     opts = list()
 802                     for itm in ctrl_panel.get(f"{c_sel['parameter']}-opt"):
 803                         set_val = ctrl_panel.get(f"{c_sel['parameter']}-val")
 804                         if isinstance(set_val, list):
 805                             if itm["value"] not in set_val:
 806                                 opts.append(itm)
 807                         else:
 808                             if itm["value"] != set_val:
 809                                 opts.append(itm)
 810                     ctrl_panel.set({
 811                         "cmp-val-opt": opts,
 812                         "cmp-val-dis": False,
 813                         "cmp-val-val": c_sel["value"]
 814                     })
 815                     on_draw = True
 816             elif trigger.type == "normalize":
 817                 ctrl_panel.set({"normalize-val": normalize})
 818                 on_draw = True
 819             elif trigger.type == "outliers":
 820                 ctrl_panel.set({"outliers-val": outliers})
 821                 on_draw = True
 822             elif trigger.type == "ctrl-dd":
 823                 if trigger.idx == "dut":
 824                     try:
 825                         opts = generate_options(self._tbs[trigger.value].keys())
 826                         disabled = False
 827                     except KeyError:
 828                         opts = list()
 829                         disabled = True
 830                     ctrl_panel.set({
 831                         "dut-val": trigger.value,
 832                         "dutver-opt": opts,
 833                         "dutver-dis": disabled,
 834                         "dutver-val": str(),
 835                         "infra-opt": list(),
 836                         "infra-dis": True,
 837                         "infra-val": str(),
 838                         "core-opt": list(),
 839                         "core-val": list(),
 840                         "frmsize-opt": list(),
 841                         "frmsize-val": list(),
 842                         "ttype-opt": list(),
 843                         "ttype-val": list(),
 844                         "cmp-par-opt": list(),
 845                         "cmp-par-dis": True,
 846                         "cmp-par-val": str(),
 847                         "cmp-val-opt": list(),
 848                         "cmp-val-dis": True,
 849                         "cmp-val-val": str()
 850                     })
 851                 elif trigger.idx == "dutver":
 852                     try:
 853                         dut = ctrl_panel.get("dut-val")
 854                         dver = self._tbs[dut][trigger.value]
 855                         opts = generate_options(dver.keys())
 856                         disabled = False
 857                     except KeyError:
 858                         opts = list()
 859                         disabled = True
 860                     ctrl_panel.set({
 861                         "dutver-val": trigger.value,
 862                         "infra-opt": opts,
 863                         "infra-dis": disabled,
 864                         "infra-val": str(),
 865                         "core-opt": list(),
 866                         "core-val": list(),
 867                         "frmsize-opt": list(),
 868                         "frmsize-val": list(),
 869                         "ttype-opt": list(),
 870                         "ttype-val": list(),
 871                         "cmp-par-opt": list(),
 872                         "cmp-par-dis": True,
 873                         "cmp-par-val": str(),
 874                         "cmp-val-opt": list(),
 875                         "cmp-val-dis": True,
 876                         "cmp-val-val": str()
 877                     })
 878                 elif trigger.idx == "infra":
 879                     dut = ctrl_panel.get("dut-val")
 880                     dver = ctrl_panel.get("dutver-val")
 881                     if all((dut, dver, trigger.value, )):
 882                         driver = self._tbs[dut][dver][trigger.value]
 883                         ctrl_panel.set({
 884                             "infra-val": trigger.value,
 885                             "core-opt": generate_options(driver["core"]),
 886                             "core-val": list(),
 887                             "frmsize-opt": generate_options(driver["fsize"]),
 888                             "frmsize-val": list(),
 889                             "ttype-opt": generate_options(driver["ttype"]),
 890                             "ttype-val": list(),
 891                             "cmp-par-opt": list(),
 892                             "cmp-par-dis": True,
 893                             "cmp-par-val": str(),
 894                             "cmp-val-opt": list(),
 895                             "cmp-val-dis": True,
 896                             "cmp-val-val": str()
 897                         })
 898                 elif trigger.idx == "cmpprm":
 899                     value = trigger.value
 900                     opts = list()
 901                     for itm in ctrl_panel.get(f"{value}-opt"):
 902                         set_val = ctrl_panel.get(f"{value}-val")
 903                         if isinstance(set_val, list):
 904                             if itm["value"] == "Latency":
 905                                 continue
 906                             if itm["value"] not in set_val:
 907                                 opts.append(itm)
 908                         else:
 909                             if itm["value"] != set_val:
 910                                 opts.append(itm)
 911                     ctrl_panel.set({
 912                         "cmp-par-val": value,
 913                         "cmp-val-opt": opts,
 914                         "cmp-val-dis": False,
 915                         "cmp-val-val": str()
 916                     })
 917                 elif trigger.idx == "cmpval":
 918                     ctrl_panel.set({"cmp-val-val": trigger.value})
 919                     selected["reference"] = {
 920                         "set": True,
 921                         "selection": {
 922                             "dut": ctrl_panel.get("dut-val"),
 923                             "dutver": ctrl_panel.get("dutver-val"),
 924                             "infra": ctrl_panel.get("infra-val"),
 925                             "core": ctrl_panel.get("core-val"),
 926                             "frmsize": ctrl_panel.get("frmsize-val"),
 927                             "ttype": ctrl_panel.get("ttype-val")
 928                         }
 929                     }
 930                     selected["compare"] = {
 931                         "set": True,
 932                         "parameter": ctrl_panel.get("cmp-par-val"),
 933                         "value": trigger.value
 934                     }
 935                     on_draw = True
 936             elif trigger.type == "ctrl-cl":
 937                 ctrl_panel.set({f"{trigger.idx}-val": trigger.value})
 938                 if all((ctrl_panel.get("core-val"),
 939                         ctrl_panel.get("frmsize-val"),
 940                         ctrl_panel.get("ttype-val"), )):
 941                     if "Latency" in ctrl_panel.get("ttype-val"):
 942                         ctrl_panel.set({"ttype-val": ["Latency", ]})
 943                     opts = list()
 944                     for itm, label in CMP_PARAMS.items():
 945                         if "Latency" in ctrl_panel.get("ttype-val") and \
 946                                 itm == "ttype":
 947                             continue
 948                         if len(ctrl_panel.get(f"{itm}-opt")) > 1:
 949                             if isinstance(ctrl_panel.get(f"{itm}-val"), list):
 950                                 if len(ctrl_panel.get(f"{itm}-opt")) == \
 951                                         len(ctrl_panel.get(f"{itm}-val")):
 952                                     continue
 953                             opts.append({"label": label, "value": itm})
 954                     ctrl_panel.set({
 955                         "cmp-par-opt": opts,
 956                         "cmp-par-dis": False,
 957                         "cmp-par-val": str(),
 958                         "cmp-val-opt": list(),
 959                         "cmp-val-dis": True,
 960                         "cmp-val-val": str()
 961                     })
 962                 else:
 963                     ctrl_panel.set({
 964                         "cmp-par-opt": list(),
 965                         "cmp-par-dis": True,
 966                         "cmp-par-val": str(),
 967                         "cmp-val-opt": list(),
 968                         "cmp-val-dis": True,
 969                         "cmp-val-val": str()
 970                     })
 971             elif trigger.type == "table" and trigger.idx == "comparison":
 972                 filtered_data = filter_table_data(
 973                     store_table_data,
 974                     table_filter[0]
 975                 )
 976                 table_data = [filtered_data, ]
 977
 978             if all((on_draw, selected["reference"]["set"],
 979                     selected["compare"]["set"], )):
 980                 title, table = comparison_table(
 981                     data=self._data,
 982                     selected=selected,
 983                     normalize=normalize,
 984                     format="html",
 985                     remove_outliers=outliers
 986                 )
 987                 plotting_area = self._get_plotting_area(
 988                     title=title,
 989                     table=table,
 990                     url=gen_new_url(
 991                         parsed_url,
 992                         params={
 993                             "selected": selected,
 994                             "norm": normalize,
 995                             "outliers": outliers
 996                         }
 997                     )
 998                 )
 999                 store_table_data = table.to_dict("records")
1000                 filtered_data = store_table_data
1001                 if table_data:
1002                     table_data = [store_table_data, ]
1003
1004             ret_val = [
1005                 ctrl_panel.panel,
1006                 selected,
1007                 store_table_data,
1008                 filtered_data,
1009                 plotting_area,
1010                 table_data
1011             ]
1012             ret_val.extend(ctrl_panel.values)
1013             return ret_val
1014
1015         @app.callback(
1016             Output("plot-mod-url", "is_open"),
1017             Input("plot-btn-url", "n_clicks"),
1018             State("plot-mod-url", "is_open")
1019         )
1020         def toggle_plot_mod_url(n, is_open):
1021             """Toggle the modal window with url.
1022             """
1023             if n:
1024                 return not is_open
1025             return is_open
1026
1027         @app.callback(
1028             Output("download-iterative-data", "data"),
1029             State("store-table-data", "data"),
1030             State("store-filtered-table-data", "data"),
1031             Input("plot-btn-download", "n_clicks"),
1032             prevent_initial_call=True
1033         )
1034         def _download_comparison_data(
1035                 table_data: list,
1036                 filtered_table_data: list,
1037                 _: int
1038             ) -> dict:
1039             """Download the data.
1040
1041             :param table_data: Original unfiltered table data.
1042             :param filtered_table_data: Filtered table data.
1043             :type table_data: list
1044             :type filtered_table_data: list
1045             :returns: dict of data frame content (base64 encoded) and meta data
1046                 used by the Download component.
1047             :rtype: dict
1048             """
1049
1050             if not table_data:
1051                 raise PreventUpdate
1052
1053             if filtered_table_data:
1054                 table = pd.DataFrame.from_records(filtered_table_data)
1055             else:
1056                 table = pd.DataFrame.from_records(table_data)
1057
1058             return dcc.send_data_frame(table.to_csv, C.COMP_DOWNLOAD_FILE_NAME)
1059
1060         @app.callback(
1061             Output("download-raw-data", "data"),
1062             State("store-selected", "data"),
1063             Input("plot-btn-download-raw", "n_clicks"),
1064             prevent_initial_call=True
1065         )
1066         def _download_raw_comparison_data(selected: dict, _: int) -> dict:
1067             """Download the data.
1068
1069             :param selected: Selected tests.
1070             :type selected: dict
1071             :returns: dict of data frame content (base64 encoded) and meta data
1072                 used by the Download component.
1073             :rtype: dict
1074             """
1075
1076             if not selected:
1077                 raise PreventUpdate
1078
1079             _, table = comparison_table(
1080                     data=self._data,
1081                     selected=selected,
1082                     normalize=False,
1083                     remove_outliers=False,
1084                     raw_data=True
1085                 )
1086
1087             return dcc.send_data_frame(
1088                 table.dropna(how="all", axis=1).to_csv,
1089                 f"raw_{C.COMP_DOWNLOAD_FILE_NAME}"
1090             )
1091
1092         @app.callback(
1093             Output("offcanvas-documentation", "is_open"),
1094             Input("btn-documentation", "n_clicks"),
1095             State("offcanvas-documentation", "is_open")
1096         )
1097         def toggle_offcanvas_documentation(n_clicks, is_open):
1098             if n_clicks:
1099                 return not is_open
1100             return is_open