csit.infra.dash/app/cdash/comparisons/layout.py

   1 # Copyright (c) 2024 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17
  18 import logging
  19 import pandas as pd
  20 import dash_bootstrap_components as dbc
  21
  22 from flask import Flask
  23 from dash import dcc, html, dash_table, callback_context, no_update, ALL
  24 from dash import Input, Output, State
  25 from dash.exceptions import PreventUpdate
  26 from dash.dash_table.Format import Format, Scheme
  27 from ast import literal_eval
  28 from yaml import load, FullLoader, YAMLError
  29
  30 from ..utils.constants import Constants as C
  31 from ..utils.control_panel import ControlPanel
  32 from ..utils.trigger import Trigger
  33 from ..utils.url_processing import url_decode
  34 from ..utils.utils import generate_options, gen_new_url, navbar_report, \
  35     filter_table_data, show_tooltip
  36 from .tables import comparison_table
  37
  38
  39 # Control panel partameters and their default values.
  40 CP_PARAMS = {
  41     "dut-val": str(),
  42     "dutver-opt": list(),
  43     "dutver-dis": True,
  44     "dutver-val": str(),
  45     "infra-opt": list(),
  46     "infra-dis": True,
  47     "infra-val": str(),
  48     "core-opt": list(),
  49     "core-val": list(),
  50     "frmsize-opt": list(),
  51     "frmsize-val": list(),
  52     "ttype-opt": list(),
  53     "ttype-val": list(),
  54     "cmp-par-opt": list(),
  55     "cmp-par-dis": True,
  56     "cmp-par-val": str(),
  57     "cmp-val-opt": list(),
  58     "cmp-val-dis": True,
  59     "cmp-val-val": str(),
  60     "normalize-val": list(),
  61     "outliers-val": list()
  62 }
  63
  64 # List of comparable parameters.
  65 CMP_PARAMS = {
  66     "dutver": "Release and Version",
  67     "infra": "Infrastructure",
  68     "frmsize": "Frame Size",
  69     "core": "Number of Cores",
  70     "ttype": "Measurement"
  71 }
  72
  73
  74 class Layout:
  75     """The layout of the dash app and the callbacks.
  76     """
  77
  78     def __init__(
  79             self,
  80             app: Flask,
  81             data_iterative: pd.DataFrame,
  82             html_layout_file: str,
  83             tooltip_file: str
  84         ) -> None:
  85         """Initialization:
  86         - save the input parameters,
  87         - prepare data for the control panel,
  88         - read HTML layout file,
  89
  90         :param app: Flask application running the dash application.
  91         :param data_iterative: Iterative data to be used in comparison tables.
  92         :param html_layout_file: Path and name of the file specifying the HTML
  93             layout of the dash application.
  94         :param tooltip_file: Path and name of the yaml file specifying the
  95             tooltips.
  96         :type app: Flask
  97         :type data_iterative: pandas.DataFrame
  98         :type html_layout_file: str
  99         :type tooltip_file: str
 100         """
 101
 102         # Inputs
 103         self._app = app
 104         self._data = data_iterative
 105         self._html_layout_file = html_layout_file
 106         self._tooltip_file = tooltip_file
 107
 108         # Get structure of tests:
 109         tbs = dict()
 110         cols = [
 111             "job", "test_id", "test_type", "dut_type", "dut_version", "tg_type",
 112             "release", "passed"
 113         ]
 114         for _, row in self._data[cols].drop_duplicates().iterrows():
 115             lst_job = row["job"].split("-")
 116             dut = lst_job[1]
 117             dver = f"{row['release']}-{row['dut_version']}"
 118             tbed = "-".join(lst_job[-2:])
 119             lst_test_id = row["test_id"].split(".")
 120
 121             suite = lst_test_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
 122                 replace("2n-", "")
 123             test = lst_test_id[-1]
 124             nic = suite.split("-")[0]
 125             for driver in C.DRIVERS:
 126                 if driver in test:
 127                     drv = driver.replace("-", "_")
 128                     test = test.replace(f"{driver}-", "")
 129                     break
 130             else:
 131                 drv = "dpdk"
 132             infra = "-".join((tbed, nic, drv))
 133             lst_test = test.split("-")
 134             fsize = lst_test[0]
 135             core = lst_test[1] if lst_test[1] else "8C"
 136
 137             if tbs.get(dut, None) is None:
 138                 tbs[dut] = dict()
 139             if tbs[dut].get(dver, None) is None:
 140                 tbs[dut][dver] = dict()
 141             if tbs[dut][dver].get(infra, None) is None:
 142                 tbs[dut][dver][infra] = dict()
 143                 tbs[dut][dver][infra]["core"] = list()
 144                 tbs[dut][dver][infra]["fsize"] = list()
 145                 tbs[dut][dver][infra]["ttype"] = list()
 146             if core.upper() not in tbs[dut][dver][infra]["core"]:
 147                 tbs[dut][dver][infra]["core"].append(core.upper())
 148             if fsize.upper() not in tbs[dut][dver][infra]["fsize"]:
 149                 tbs[dut][dver][infra]["fsize"].append(fsize.upper())
 150             if row["test_type"] == "mrr":
 151                 if "MRR" not in tbs[dut][dver][infra]["ttype"]:
 152                     tbs[dut][dver][infra]["ttype"].append("MRR")
 153             elif row["test_type"] == "ndrpdr":
 154                 if "NDR" not in tbs[dut][dver][infra]["ttype"]:
 155                     tbs[dut][dver][infra]["ttype"].extend(
 156                         ("NDR", "PDR", "Latency")
 157                     )
 158             elif row["test_type"] == "hoststack" and \
 159                     row["tg_type"] in ("iperf", "vpp"):
 160                 if "BPS" not in tbs[dut][dver][infra]["ttype"]:
 161                     tbs[dut][dver][infra]["ttype"].append("BPS")
 162             elif row["test_type"] == "hoststack" and row["tg_type"] == "ab":
 163                 if "CPS" not in tbs[dut][dver][infra]["ttype"]:
 164                     tbs[dut][dver][infra]["ttype"].extend(("CPS", "RPS", ))
 165         self._tbs = tbs
 166
 167         # Read from files:
 168         self._html_layout = str()
 169         try:
 170             with open(self._html_layout_file, "r") as file_read:
 171                 self._html_layout = file_read.read()
 172         except IOError as err:
 173             raise RuntimeError(
 174                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 175             )
 176
 177         try:
 178             with open(self._tooltip_file, "r") as file_read:
 179                 self._tooltips = load(file_read, Loader=FullLoader)
 180         except IOError as err:
 181             logging.warning(
 182                 f"Not possible to open the file {self._tooltip_file}\n{err}"
 183             )
 184         except YAMLError as err:
 185             logging.warning(
 186                 f"An error occurred while parsing the specification file "
 187                 f"{self._tooltip_file}\n{err}"
 188             )
 189
 190         # Callbacks:
 191         if self._app is not None and hasattr(self, "callbacks"):
 192             self.callbacks(self._app)
 193
 194     @property
 195     def html_layout(self):
 196         return self._html_layout
 197
 198     def add_content(self):
 199         """Top level method which generated the web page.
 200
 201         It generates:
 202         - Store for user input data,
 203         - Navigation bar,
 204         - Main area with control panel and ploting area.
 205
 206         If no HTML layout is provided, an error message is displayed instead.
 207
 208         :returns: The HTML div with the whole page.
 209         :rtype: html.Div
 210         """
 211
 212         if self.html_layout and self._tbs:
 213             return html.Div(
 214                 id="div-main",
 215                 className="small",
 216                 children=[
 217                     dbc.Row(
 218                         id="row-navbar",
 219                         class_name="g-0",
 220                         children=[navbar_report((False, True, False, False)), ]
 221                     ),
 222                     dbc.Row(
 223                         id="row-main",
 224                         class_name="g-0",
 225                         children=[
 226                             dcc.Store(id="store-control-panel"),
 227                             dcc.Store(id="store-selected"),
 228                             dcc.Store(id="store-table-data"),
 229                             dcc.Store(id="store-filtered-table-data"),
 230                             dcc.Location(id="url", refresh=False),
 231                             self._add_ctrl_col(),
 232                             self._add_plotting_col()
 233                         ]
 234                     ),
 235                     dbc.Offcanvas(
 236                         class_name="w-75",
 237                         id="offcanvas-documentation",
 238                         title="Documentation",
 239                         placement="end",
 240                         is_open=False,
 241                         children=html.Iframe(
 242                             src=C.URL_DOC_REL_NOTES,
 243                             width="100%",
 244                             height="100%"
 245                         )
 246                     )
 247                 ]
 248             )
 249         else:
 250             return html.Div(
 251                 id="div-main-error",
 252                 children=[
 253                     dbc.Alert(
 254                         [
 255                             "An Error Occured"
 256                         ],
 257                         color="danger"
 258                     )
 259                 ]
 260             )
 261
 262     def _add_ctrl_col(self) -> dbc.Col:
 263         """Add column with controls. It is placed on the left side.
 264
 265         :returns: Column with the control panel.
 266         :rtype: dbc.Col
 267         """
 268         return dbc.Col([
 269             html.Div(
 270                 children=self._add_ctrl_panel(),
 271                 className="sticky-top"
 272             )
 273         ])
 274
 275     def _add_plotting_col(self) -> dbc.Col:
 276         """Add column with plots. It is placed on the right side.
 277
 278         :returns: Column with plots.
 279         :rtype: dbc.Col
 280         """
 281         return dbc.Col(
 282             id="col-plotting-area",
 283             children=[
 284                 dbc.Spinner(
 285                     children=[
 286                         dbc.Row(
 287                             id="plotting-area",
 288                             class_name="g-0 p-0",
 289                             children=[
 290                                 C.PLACEHOLDER
 291                             ]
 292                         )
 293                     ]
 294                 )
 295             ],
 296             width=9
 297         )
 298
 299     def _add_ctrl_panel(self) -> list:
 300         """Add control panel.
 301
 302         :returns: Control panel.
 303         :rtype: list
 304         """
 305
 306         reference = [
 307             dbc.Row(
 308                 class_name="g-0 p-1",
 309                 children=[
 310                     dbc.InputGroup(
 311                         [
 312                             dbc.InputGroupText(
 313                                 show_tooltip(self._tooltips, "help-dut", "DUT")
 314                             ),
 315                             dbc.Select(
 316                                 id={"type": "ctrl-dd", "index": "dut"},
 317                                 placeholder="Select a Device under Test...",
 318                                 options=sorted(
 319                                     [
 320                                         {"label": k, "value": k} \
 321                                             for k in self._tbs.keys()
 322                                     ],
 323                                     key=lambda d: d["label"]
 324                                 )
 325                             )
 326                         ],
 327                         size="sm"
 328                     )
 329                 ]
 330             ),
 331             dbc.Row(
 332                 class_name="g-0 p-1",
 333                 children=[
 334                     dbc.InputGroup(
 335                         [
 336                             dbc.InputGroupText(show_tooltip(
 337                                 self._tooltips,
 338                                 "help-csit-dut",
 339                                 "CSIT and DUT Version"
 340                             )),
 341                             dbc.Select(
 342                                 id={"type": "ctrl-dd", "index": "dutver"},
 343                                 placeholder="Select a CSIT and DUT Version...")
 344                         ],
 345                         size="sm"
 346                     )
 347                 ]
 348             ),
 349             dbc.Row(
 350                 class_name="g-0 p-1",
 351                 children=[
 352                     dbc.InputGroup(
 353                         [
 354                             dbc.InputGroupText(show_tooltip(
 355                                 self._tooltips,
 356                                 "help-infra",
 357                                 "Infra"
 358                             )),
 359                             dbc.Select(
 360                                 id={"type": "ctrl-dd", "index": "infra"},
 361                                 placeholder=\
 362                                     "Select a Physical Test Bed Topology..."
 363                             )
 364                         ],
 365                         size="sm"
 366                     )
 367                 ]
 368             ),
 369             dbc.Row(
 370                 class_name="g-0 p-1",
 371                 children=[
 372                     dbc.InputGroup(
 373                         [
 374                             dbc.InputGroupText(show_tooltip(
 375                                 self._tooltips,
 376                                 "help-framesize",
 377                                 "Frame Size"
 378                             )),
 379                             dbc.Checklist(
 380                                 id={"type": "ctrl-cl", "index": "frmsize"},
 381                                 inline=True,
 382                                 class_name="ms-2"
 383                             )
 384                         ],
 385                         style={"align-items": "center"},
 386                         size="sm"
 387                     )
 388                 ]
 389             ),
 390             dbc.Row(
 391                 class_name="g-0 p-1",
 392                 children=[
 393                     dbc.InputGroup(
 394                         [
 395                             dbc.InputGroupText(show_tooltip(
 396                                 self._tooltips,
 397                                 "help-cores",
 398                                 "Number of Cores"
 399                             )),
 400                             dbc.Checklist(
 401                                 id={"type": "ctrl-cl", "index": "core"},
 402                                 inline=True,
 403                                 class_name="ms-2"
 404                             )
 405                         ],
 406                         style={"align-items": "center"},
 407                         size="sm"
 408                     )
 409                 ]
 410             ),
 411             dbc.Row(
 412                 class_name="g-0 p-1",
 413                 children=[
 414                     dbc.InputGroup(
 415                         [
 416                             dbc.InputGroupText(show_tooltip(
 417                                 self._tooltips,
 418                                 "help-measurement",
 419                                 "Measurement"
 420                             )),
 421                             dbc.Checklist(
 422                                 id={"type": "ctrl-cl", "index": "ttype"},
 423                                 inline=True,
 424                                 class_name="ms-2"
 425                             )
 426                         ],
 427                         style={"align-items": "center"},
 428                         size="sm"
 429                     )
 430                 ]
 431             )
 432         ]
 433
 434         compare = [
 435             dbc.Row(
 436                 class_name="g-0 p-1",
 437                 children=[
 438                     dbc.InputGroup(
 439                         [
 440                             dbc.InputGroupText(show_tooltip(
 441                                 self._tooltips,
 442                                 "help-cmp-parameter",
 443                                 "Parameter"
 444                             )),
 445                             dbc.Select(
 446                                 id={"type": "ctrl-dd", "index": "cmpprm"},
 447                                 placeholder="Select a Parameter..."
 448                             )
 449                         ],
 450                         size="sm"
 451                     )
 452                 ]
 453             ),
 454             dbc.Row(
 455                 class_name="g-0 p-1",
 456                 children=[
 457                     dbc.InputGroup(
 458                         [
 459                             dbc.InputGroupText(show_tooltip(
 460                                 self._tooltips,
 461                                 "help-cmp-value",
 462                                 "Value"
 463                             )),
 464                             dbc.Select(
 465                                 id={"type": "ctrl-dd", "index": "cmpval"},
 466                                 placeholder="Select a Value..."
 467                             )
 468                         ],
 469                         size="sm"
 470                     )
 471                 ]
 472             )
 473         ]
 474
 475         processing = [
 476             dbc.Row(
 477                 class_name="g-0 p-1",
 478                 children=[
 479                     dbc.InputGroup(
 480                         children = [
 481                             dbc.Checklist(
 482                                 id="normalize",
 483                                 options=[{
 484                                     "value": "normalize",
 485                                     "label": "Normalize to 2GHz CPU frequency"
 486                                 }],
 487                                 value=[],
 488                                 inline=True,
 489                                 class_name="ms-2"
 490                             ),
 491                             dbc.Checklist(
 492                                 id="outliers",
 493                                 options=[{
 494                                     "value": "outliers",
 495                                     "label": "Remove Extreme Outliers"
 496                                 }],
 497                                 value=[],
 498                                 inline=True,
 499                                 class_name="ms-2"
 500                             )
 501                         ],
 502                         style={"align-items": "center"},
 503                         size="sm"
 504                     )
 505                 ]
 506             )
 507         ]
 508
 509         return [
 510             dbc.Row(
 511                 dbc.Card(
 512                     [
 513                         dbc.CardHeader(
 514                             html.H5("Reference Value")
 515                         ),
 516                         dbc.CardBody(
 517                             children=reference,
 518                             class_name="g-0 p-0"
 519                         )
 520                     ],
 521                     color="secondary",
 522                     outline=True
 523                 ),
 524                 class_name="g-0 p-1"
 525             ),
 526             dbc.Row(
 527                 dbc.Card(
 528                     [
 529                         dbc.CardHeader(
 530                             html.H5("Compared Value")
 531                         ),
 532                         dbc.CardBody(
 533                             children=compare,
 534                             class_name="g-0 p-0"
 535                         )
 536                     ],
 537                     color="secondary",
 538                     outline=True
 539                 ),
 540                 class_name="g-0 p-1"
 541             ),
 542             dbc.Row(
 543                 dbc.Card(
 544                     [
 545                         dbc.CardHeader(
 546                             html.H5("Data Manipulations")
 547                         ),
 548                         dbc.CardBody(
 549                             children=processing,
 550                             class_name="g-0 p-0"
 551                         )
 552                     ],
 553                     color="secondary",
 554                     outline=True
 555                 ),
 556                 class_name="g-0 p-1"
 557             )
 558         ]
 559
 560     @staticmethod
 561     def _get_plotting_area(
 562             title: str,
 563             table: pd.DataFrame,
 564             url: str
 565         ) -> list:
 566         """Generate the plotting area with all its content.
 567
 568         :param title: The title of the comparison table.
 569         :param table: Comparison table to be displayed.
 570         :param url: URL to be displayed in the modal window.
 571         :type title: str
 572         :type table: pandas.DataFrame
 573         :type url: str
 574         :returns: List of rows with elements to be displayed in the plotting
 575             area.
 576         :rtype: list
 577         """
 578
 579         if table.empty:
 580             return dbc.Row(
 581                 dbc.Col(
 582                     children=dbc.Alert(
 583                         "No data for comparison.",
 584                         color="danger"
 585                     ),
 586                     class_name="g-0 p-1",
 587                 ),
 588                 class_name="g-0 p-0"
 589             )
 590
 591         cols = list()
 592         for idx, col in enumerate(table.columns):
 593             if idx == 0:
 594                 cols.append({
 595                     "name": ["", col],
 596                     "id": col,
 597                     "deletable": False,
 598                     "selectable": False,
 599                     "type": "text"
 600                 })
 601             else:
 602                 l_col = col.rsplit(" ", 2)
 603                 cols.append({
 604                     "name": [l_col[0], " ".join(l_col[-2:])],
 605                     "id": col,
 606                     "deletable": False,
 607                     "selectable": False,
 608                     "type": "numeric",
 609                     "format": Format(precision=2, scheme=Scheme.fixed)
 610                 })
 611
 612         return [
 613             dbc.Row(
 614                 children=html.H5(title),
 615                 class_name="g-0 p-1"
 616             ),
 617             dbc.Row(
 618                 children=[
 619                     dbc.Col(
 620                         children=dash_table.DataTable(
 621                             id={"type": "table", "index": "comparison"},
 622                             columns=cols,
 623                             data=table.to_dict("records"),
 624                             merge_duplicate_headers=True,
 625                             editable=False,
 626                             filter_action="custom",
 627                             filter_query="",
 628                             sort_action="native",
 629                             sort_mode="multi",
 630                             selected_columns=[],
 631                             selected_rows=[],
 632                             page_action="none",
 633                             style_cell={"textAlign": "right"},
 634                             style_cell_conditional=[{
 635                                 "if": {"column_id": "Test Name"},
 636                                 "textAlign": "left"
 637                             }]
 638                         ),
 639                         class_name="g-0 p-1"
 640                     )
 641                 ],
 642                 class_name="g-0 p-0"
 643             ),
 644             dbc.Row(
 645                 [
 646                     dbc.Col([html.Div(
 647                         [
 648                             dbc.Button(
 649                                 id="plot-btn-url",
 650                                 children="Show URL",
 651                                 class_name="me-1",
 652                                 color="info",
 653                                 style={
 654                                     "text-transform": "none",
 655                                     "padding": "0rem 1rem"
 656                                 }
 657                             ),
 658                             dbc.Modal(
 659                                 [
 660                                     dbc.ModalHeader(dbc.ModalTitle("URL")),
 661                                     dbc.ModalBody(url)
 662                                 ],
 663                                 id="plot-mod-url",
 664                                 size="xl",
 665                                 is_open=False,
 666                                 scrollable=True
 667                             ),
 668                             dbc.Button(
 669                                 id="plot-btn-download",
 670                                 children="Download Table",
 671                                 class_name="me-1",
 672                                 color="info",
 673                                 style={
 674                                     "text-transform": "none",
 675                                     "padding": "0rem 1rem"
 676                                 }
 677                             ),
 678                             dcc.Download(id="download-iterative-data"),
 679                             dbc.Button(
 680                                 id="plot-btn-download-raw",
 681                                 children="Download Raw Data",
 682                                 class_name="me-1",
 683                                 color="info",
 684                                 style={
 685                                     "text-transform": "none",
 686                                     "padding": "0rem 1rem"
 687                                 }
 688                             ),
 689                             dcc.Download(id="download-raw-data")
 690                         ],
 691                         className=\
 692                             "d-grid gap-0 d-md-flex justify-content-md-end"
 693                     )])
 694                 ],
 695                 class_name="g-0 p-0"
 696             ),
 697             dbc.Row(
 698                 children=C.PLACEHOLDER,
 699                 class_name="g-0 p-1"
 700             )
 701         ]
 702
 703     def callbacks(self, app):
 704         """Callbacks for the whole application.
 705
 706         :param app: The application.
 707         :type app: Flask
 708         """
 709
 710         @app.callback(
 711             [
 712                 Output("store-control-panel", "data"),
 713                 Output("store-selected", "data"),
 714                 Output("store-table-data", "data"),
 715                 Output("store-filtered-table-data", "data"),
 716                 Output("plotting-area", "children"),
 717                 Output({"type": "table", "index": ALL}, "data"),
 718                 Output({"type": "ctrl-dd", "index": "dut"}, "value"),
 719                 Output({"type": "ctrl-dd", "index": "dutver"}, "options"),
 720                 Output({"type": "ctrl-dd", "index": "dutver"}, "disabled"),
 721                 Output({"type": "ctrl-dd", "index": "dutver"}, "value"),
 722                 Output({"type": "ctrl-dd", "index": "infra"}, "options"),
 723                 Output({"type": "ctrl-dd", "index": "infra"}, "disabled"),
 724                 Output({"type": "ctrl-dd", "index": "infra"}, "value"),
 725                 Output({"type": "ctrl-cl", "index": "core"}, "options"),
 726                 Output({"type": "ctrl-cl", "index": "core"}, "value"),
 727                 Output({"type": "ctrl-cl", "index": "frmsize"}, "options"),
 728                 Output({"type": "ctrl-cl", "index": "frmsize"}, "value"),
 729                 Output({"type": "ctrl-cl", "index": "ttype"}, "options"),
 730                 Output({"type": "ctrl-cl", "index": "ttype"}, "value"),
 731                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "options"),
 732                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "disabled"),
 733                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "value"),
 734                 Output({"type": "ctrl-dd", "index": "cmpval"}, "options"),
 735                 Output({"type": "ctrl-dd", "index": "cmpval"}, "disabled"),
 736                 Output({"type": "ctrl-dd", "index": "cmpval"}, "value"),
 737                 Output("normalize", "value"),
 738                 Output("outliers", "value")
 739             ],
 740             [
 741                 State("store-control-panel", "data"),
 742                 State("store-selected", "data"),
 743                 State("store-table-data", "data"),
 744                 State("store-filtered-table-data", "data"),
 745                 State({"type": "table", "index": ALL}, "data")
 746             ],
 747             [
 748                 Input("url", "href"),
 749                 Input("normalize", "value"),
 750                 Input("outliers", "value"),
 751                 Input({"type": "table", "index": ALL}, "filter_query"),
 752                 Input({"type": "ctrl-dd", "index": ALL}, "value"),
 753                 Input({"type": "ctrl-cl", "index": ALL}, "value"),
 754                 Input({"type": "ctrl-btn", "index": ALL}, "n_clicks")
 755             ]
 756         )
 757         def _update_application(
 758                 control_panel: dict,
 759                 selected: dict,
 760                 store_table_data: list,
 761                 filtered_data: list,
 762                 table_data: list,
 763                 href: str,
 764                 normalize: list,
 765                 outliers: bool,
 766                 table_filter: str,
 767                 *_
 768             ) -> tuple:
 769             """Update the application when the event is detected.
 770             """
 771
 772             ctrl_panel = ControlPanel(CP_PARAMS, control_panel)
 773
 774             if selected is None:
 775                 selected = {
 776                     "reference": {
 777                         "set": False,
 778                     },
 779                     "compare": {
 780                         "set": False,
 781                     }
 782                 }
 783
 784             # Parse the url:
 785             parsed_url = url_decode(href)
 786             if parsed_url:
 787                 url_params = parsed_url["params"]
 788             else:
 789                 url_params = None
 790
 791             on_draw = False
 792             plotting_area = no_update
 793
 794             trigger = Trigger(callback_context.triggered)
 795             if trigger.type == "url" and url_params:
 796                 process_url = False
 797                 try:
 798                     selected = literal_eval(url_params["selected"][0])
 799                     r_sel = selected["reference"]["selection"]
 800                     c_sel = selected["compare"]
 801                     normalize = literal_eval(url_params["norm"][0])
 802                     try:  # Necessary for backward compatibility
 803                         outliers = literal_eval(url_params["outliers"][0])
 804                     except (KeyError, IndexError, AttributeError):
 805                         outliers = list()
 806                     process_url = bool(
 807                         (selected["reference"]["set"] == True) and
 808                         (c_sel["set"] == True)
 809                     )
 810                 except (KeyError, IndexError, AttributeError):
 811                     pass
 812                 if process_url:
 813                     ctrl_panel.set({
 814                         "dut-val": r_sel["dut"],
 815                         "dutver-opt": generate_options(
 816                             self._tbs[r_sel["dut"]].keys()
 817                         ),
 818                         "dutver-dis": False,
 819                         "dutver-val": r_sel["dutver"],
 820                         "infra-opt": generate_options(
 821                             self._tbs[r_sel["dut"]][r_sel["dutver"]].keys()
 822                         ),
 823                         "infra-dis": False,
 824                         "infra-val": r_sel["infra"],
 825                         "core-opt": generate_options(
 826                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 827                                 [r_sel["infra"]]["core"]
 828                         ),
 829                         "core-val": r_sel["core"],
 830                         "frmsize-opt": generate_options(
 831                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 832                                 [r_sel["infra"]]["fsize"]
 833                         ),
 834                         "frmsize-val": r_sel["frmsize"],
 835                         "ttype-opt": generate_options(
 836                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 837                                 [r_sel["infra"]]["ttype"]
 838                         ),
 839                         "ttype-val": r_sel["ttype"],
 840                         "normalize-val": normalize,
 841                         "outliers-val": outliers
 842                     })
 843                     opts = list()
 844                     for itm, label in CMP_PARAMS.items():
 845                         if len(ctrl_panel.get(f"{itm}-opt")) > 1:
 846                             opts.append({"label": label, "value": itm})
 847                     ctrl_panel.set({
 848                         "cmp-par-opt": opts,
 849                         "cmp-par-dis": False,
 850                         "cmp-par-val": c_sel["parameter"]
 851                     })
 852                     opts = list()
 853                     for itm in ctrl_panel.get(f"{c_sel['parameter']}-opt"):
 854                         set_val = ctrl_panel.get(f"{c_sel['parameter']}-val")
 855                         if isinstance(set_val, list):
 856                             if itm["value"] not in set_val:
 857                                 opts.append(itm)
 858                         else:
 859                             if itm["value"] != set_val:
 860                                 opts.append(itm)
 861                     ctrl_panel.set({
 862                         "cmp-val-opt": opts,
 863                         "cmp-val-dis": False,
 864                         "cmp-val-val": c_sel["value"]
 865                     })
 866                     on_draw = True
 867             elif trigger.type == "normalize":
 868                 ctrl_panel.set({"normalize-val": normalize})
 869                 on_draw = True
 870             elif trigger.type == "outliers":
 871                 ctrl_panel.set({"outliers-val": outliers})
 872                 on_draw = True
 873             elif trigger.type == "ctrl-dd":
 874                 if trigger.idx == "dut":
 875                     try:
 876                         opts = generate_options(self._tbs[trigger.value].keys())
 877                         disabled = False
 878                     except KeyError:
 879                         opts = list()
 880                         disabled = True
 881                     ctrl_panel.set({
 882                         "dut-val": trigger.value,
 883                         "dutver-opt": opts,
 884                         "dutver-dis": disabled,
 885                         "dutver-val": str(),
 886                         "infra-opt": list(),
 887                         "infra-dis": True,
 888                         "infra-val": str(),
 889                         "core-opt": list(),
 890                         "core-val": list(),
 891                         "frmsize-opt": list(),
 892                         "frmsize-val": list(),
 893                         "ttype-opt": list(),
 894                         "ttype-val": list(),
 895                         "cmp-par-opt": list(),
 896                         "cmp-par-dis": True,
 897                         "cmp-par-val": str(),
 898                         "cmp-val-opt": list(),
 899                         "cmp-val-dis": True,
 900                         "cmp-val-val": str()
 901                     })
 902                 elif trigger.idx == "dutver":
 903                     try:
 904                         dut = ctrl_panel.get("dut-val")
 905                         dver = self._tbs[dut][trigger.value]
 906                         opts = generate_options(dver.keys())
 907                         disabled = False
 908                     except KeyError:
 909                         opts = list()
 910                         disabled = True
 911                     ctrl_panel.set({
 912                         "dutver-val": trigger.value,
 913                         "infra-opt": opts,
 914                         "infra-dis": disabled,
 915                         "infra-val": str(),
 916                         "core-opt": list(),
 917                         "core-val": list(),
 918                         "frmsize-opt": list(),
 919                         "frmsize-val": list(),
 920                         "ttype-opt": list(),
 921                         "ttype-val": list(),
 922                         "cmp-par-opt": list(),
 923                         "cmp-par-dis": True,
 924                         "cmp-par-val": str(),
 925                         "cmp-val-opt": list(),
 926                         "cmp-val-dis": True,
 927                         "cmp-val-val": str()
 928                     })
 929                 elif trigger.idx == "infra":
 930                     dut = ctrl_panel.get("dut-val")
 931                     dver = ctrl_panel.get("dutver-val")
 932                     if all((dut, dver, trigger.value, )):
 933                         driver = self._tbs[dut][dver][trigger.value]
 934                         ctrl_panel.set({
 935                             "infra-val": trigger.value,
 936                             "core-opt": generate_options(driver["core"]),
 937                             "core-val": list(),
 938                             "frmsize-opt": generate_options(driver["fsize"]),
 939                             "frmsize-val": list(),
 940                             "ttype-opt": generate_options(driver["ttype"]),
 941                             "ttype-val": list(),
 942                             "cmp-par-opt": list(),
 943                             "cmp-par-dis": True,
 944                             "cmp-par-val": str(),
 945                             "cmp-val-opt": list(),
 946                             "cmp-val-dis": True,
 947                             "cmp-val-val": str()
 948                         })
 949                 elif trigger.idx == "cmpprm":
 950                     value = trigger.value
 951                     opts = list()
 952                     for itm in ctrl_panel.get(f"{value}-opt"):
 953                         set_val = ctrl_panel.get(f"{value}-val")
 954                         if isinstance(set_val, list):
 955                             if itm["value"] == "Latency":
 956                                 continue
 957                             if itm["value"] not in set_val:
 958                                 opts.append(itm)
 959                         else:
 960                             if itm["value"] != set_val:
 961                                 opts.append(itm)
 962                     ctrl_panel.set({
 963                         "cmp-par-val": value,
 964                         "cmp-val-opt": opts,
 965                         "cmp-val-dis": False,
 966                         "cmp-val-val": str()
 967                     })
 968                 elif trigger.idx == "cmpval":
 969                     ctrl_panel.set({"cmp-val-val": trigger.value})
 970                     selected["reference"] = {
 971                         "set": True,
 972                         "selection": {
 973                             "dut": ctrl_panel.get("dut-val"),
 974                             "dutver": ctrl_panel.get("dutver-val"),
 975                             "infra": ctrl_panel.get("infra-val"),
 976                             "core": ctrl_panel.get("core-val"),
 977                             "frmsize": ctrl_panel.get("frmsize-val"),
 978                             "ttype": ctrl_panel.get("ttype-val")
 979                         }
 980                     }
 981                     selected["compare"] = {
 982                         "set": True,
 983                         "parameter": ctrl_panel.get("cmp-par-val"),
 984                         "value": trigger.value
 985                     }
 986                     on_draw = True
 987             elif trigger.type == "ctrl-cl":
 988                 ctrl_panel.set({f"{trigger.idx}-val": trigger.value})
 989                 if all((ctrl_panel.get("core-val"),
 990                         ctrl_panel.get("frmsize-val"),
 991                         ctrl_panel.get("ttype-val"), )):
 992                     if "Latency" in ctrl_panel.get("ttype-val"):
 993                         ctrl_panel.set({"ttype-val": ["Latency", ]})
 994                     opts = list()
 995                     for itm, label in CMP_PARAMS.items():
 996                         if "Latency" in ctrl_panel.get("ttype-val") and \
 997                                 itm == "ttype":
 998                             continue
 999                         if len(ctrl_panel.get(f"{itm}-opt")) > 1:
1000                             if isinstance(ctrl_panel.get(f"{itm}-val"), list):
1001                                 if len(ctrl_panel.get(f"{itm}-opt")) == \
1002                                         len(ctrl_panel.get(f"{itm}-val")):
1003                                     continue
1004                             opts.append({"label": label, "value": itm})
1005                     ctrl_panel.set({
1006                         "cmp-par-opt": opts,
1007                         "cmp-par-dis": False,
1008                         "cmp-par-val": str(),
1009                         "cmp-val-opt": list(),
1010                         "cmp-val-dis": True,
1011                         "cmp-val-val": str()
1012                     })
1013                 else:
1014                     ctrl_panel.set({
1015                         "cmp-par-opt": list(),
1016                         "cmp-par-dis": True,
1017                         "cmp-par-val": str(),
1018                         "cmp-val-opt": list(),
1019                         "cmp-val-dis": True,
1020                         "cmp-val-val": str()
1021                     })
1022             elif trigger.type == "table" and trigger.idx == "comparison":
1023                 filtered_data = filter_table_data(
1024                     store_table_data,
1025                     table_filter[0]
1026                 )
1027                 table_data = [filtered_data, ]
1028
1029             if all((on_draw, selected["reference"]["set"],
1030                     selected["compare"]["set"], )):
1031                 title, table = comparison_table(
1032                     data=self._data,
1033                     selected=selected,
1034                     normalize=normalize,
1035                     format="html",
1036                     remove_outliers=outliers
1037                 )
1038                 plotting_area = self._get_plotting_area(
1039                     title=title,
1040                     table=table,
1041                     url=gen_new_url(
1042                         parsed_url,
1043                         params={
1044                             "selected": selected,
1045                             "norm": normalize,
1046                             "outliers": outliers
1047                         }
1048                     )
1049                 )
1050                 store_table_data = table.to_dict("records")
1051                 filtered_data = store_table_data
1052                 if table_data:
1053                     table_data = [store_table_data, ]
1054
1055             ret_val = [
1056                 ctrl_panel.panel,
1057                 selected,
1058                 store_table_data,
1059                 filtered_data,
1060                 plotting_area,
1061                 table_data
1062             ]
1063             ret_val.extend(ctrl_panel.values)
1064             return ret_val
1065
1066         @app.callback(
1067             Output("plot-mod-url", "is_open"),
1068             Input("plot-btn-url", "n_clicks"),
1069             State("plot-mod-url", "is_open")
1070         )
1071         def toggle_plot_mod_url(n, is_open):
1072             """Toggle the modal window with url.
1073             """
1074             if n:
1075                 return not is_open
1076             return is_open
1077
1078         @app.callback(
1079             Output("download-iterative-data", "data"),
1080             State("store-table-data", "data"),
1081             State("store-filtered-table-data", "data"),
1082             Input("plot-btn-download", "n_clicks"),
1083             prevent_initial_call=True
1084         )
1085         def _download_comparison_data(
1086                 table_data: list,
1087                 filtered_table_data: list,
1088                 _: int
1089             ) -> dict:
1090             """Download the data.
1091
1092             :param table_data: Original unfiltered table data.
1093             :param filtered_table_data: Filtered table data.
1094             :type table_data: list
1095             :type filtered_table_data: list
1096             :returns: dict of data frame content (base64 encoded) and meta data
1097                 used by the Download component.
1098             :rtype: dict
1099             """
1100
1101             if not table_data:
1102                 raise PreventUpdate
1103
1104             if filtered_table_data:
1105                 table = pd.DataFrame.from_records(filtered_table_data)
1106             else:
1107                 table = pd.DataFrame.from_records(table_data)
1108
1109             return dcc.send_data_frame(table.to_csv, C.COMP_DOWNLOAD_FILE_NAME)
1110
1111         @app.callback(
1112             Output("download-raw-data", "data"),
1113             State("store-selected", "data"),
1114             Input("plot-btn-download-raw", "n_clicks"),
1115             prevent_initial_call=True
1116         )
1117         def _download_raw_comparison_data(selected: dict, _: int) -> dict:
1118             """Download the data.
1119
1120             :param selected: Selected tests.
1121             :type selected: dict
1122             :returns: dict of data frame content (base64 encoded) and meta data
1123                 used by the Download component.
1124             :rtype: dict
1125             """
1126
1127             if not selected:
1128                 raise PreventUpdate
1129
1130             _, table = comparison_table(
1131                     data=self._data,
1132                     selected=selected,
1133                     normalize=False,
1134                     remove_outliers=False,
1135                     raw_data=True
1136                 )
1137
1138             return dcc.send_data_frame(
1139                 table.dropna(how="all", axis=1).to_csv,
1140                 f"raw_{C.COMP_DOWNLOAD_FILE_NAME}"
1141             )
1142
1143         @app.callback(
1144             Output("offcanvas-documentation", "is_open"),
1145             Input("btn-documentation", "n_clicks"),
1146             State("offcanvas-documentation", "is_open")
1147         )
1148         def toggle_offcanvas_documentation(n_clicks, is_open):
1149             if n_clicks:
1150                 return not is_open
1151             return is_open