csit.infra.dash/app/cdash/comparisons/layout.py

   1 # Copyright (c) 2024 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17
  18 import logging
  19 import pandas as pd
  20 import dash_bootstrap_components as dbc
  21
  22 from flask import Flask
  23 from dash import dcc, html, dash_table, callback_context, no_update, ALL
  24 from dash import Input, Output, State
  25 from dash.exceptions import PreventUpdate
  26 from dash.dash_table.Format import Format, Scheme
  27 from ast import literal_eval
  28 from yaml import load, FullLoader, YAMLError
  29 from copy import deepcopy
  30
  31 from ..utils.constants import Constants as C
  32 from ..utils.control_panel import ControlPanel
  33 from ..utils.trigger import Trigger
  34 from ..utils.url_processing import url_decode
  35 from ..utils.utils import generate_options, gen_new_url, navbar_report, \
  36     filter_table_data, sort_table_data, show_iterative_graph_data, show_tooltip
  37 from .tables import comparison_table
  38 from ..report.graphs import graph_iterative
  39
  40
  41 # Control panel partameters and their default values.
  42 CP_PARAMS = {
  43     "dut-val": str(),
  44     "dutver-opt": list(),
  45     "dutver-dis": True,
  46     "dutver-val": str(),
  47     "infra-opt": list(),
  48     "infra-dis": True,
  49     "infra-val": str(),
  50     "core-opt": list(),
  51     "core-val": list(),
  52     "frmsize-opt": list(),
  53     "frmsize-val": list(),
  54     "ttype-opt": list(),
  55     "ttype-val": list(),
  56     "cmp-par-opt": list(),
  57     "cmp-par-dis": True,
  58     "cmp-par-val": str(),
  59     "cmp-val-opt": list(),
  60     "cmp-val-dis": True,
  61     "cmp-val-val": str(),
  62     "normalize-val": list(),
  63     "outliers-val": list()
  64 }
  65
  66 # List of comparable parameters.
  67 CMP_PARAMS = {
  68     "dutver": "Release and Version",
  69     "infra": "Infrastructure",
  70     "frmsize": "Frame Size",
  71     "core": "Number of Cores",
  72     "ttype": "Measurement"
  73 }
  74
  75
  76 class Layout:
  77     """The layout of the dash app and the callbacks.
  78     """
  79
  80     def __init__(
  81             self,
  82             app: Flask,
  83             data_iterative: pd.DataFrame,
  84             html_layout_file: str,
  85             graph_layout_file: str,
  86             tooltip_file: str
  87         ) -> None:
  88         """Initialization:
  89         - save the input parameters,
  90         - prepare data for the control panel,
  91         - read HTML layout file,
  92         - read graph layout file,
  93         - read tooltips from the tooltip file.
  94
  95         :param app: Flask application running the dash application.
  96         :param data_iterative: Iterative data to be used in comparison tables.
  97         :param html_layout_file: Path and name of the file specifying the HTML
  98             layout of the dash application.
  99         :param tooltip_file: Path and name of the yaml file specifying the
 100             tooltips.
 101         :param graph_layout_file: Path and name of the file with layout of
 102             plot.ly graphs.
 103         :type app: Flask
 104         :type data_iterative: pandas.DataFrame
 105         :type html_layout_file: str
 106         :type graph_layout_file: str
 107         :type tooltip_file: str
 108         """
 109
 110         # Inputs
 111         self._app = app
 112         self._data = data_iterative
 113         self._html_layout_file = html_layout_file
 114         self._graph_layout_file = graph_layout_file
 115         self._tooltip_file = tooltip_file
 116
 117         # Get structure of tests:
 118         tbs = dict()
 119         cols = [
 120             "job", "test_id", "test_type", "dut_type", "dut_version", "tg_type",
 121             "release", "passed"
 122         ]
 123         for _, row in self._data[cols].drop_duplicates().iterrows():
 124             lst_job = row["job"].split("-")
 125             dut = lst_job[1]
 126             dver = f"{row['release']}-{row['dut_version']}"
 127             tbed = "-".join(lst_job[-2:])
 128             lst_test_id = row["test_id"].split(".")
 129
 130             suite = lst_test_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
 131                 replace("2n-", "")
 132             test = lst_test_id[-1]
 133             nic = suite.split("-")[0]
 134             for driver in C.DRIVERS:
 135                 if driver in test:
 136                     drv = driver.replace("-", "_")
 137                     test = test.replace(f"{driver}-", "")
 138                     break
 139             else:
 140                 drv = "dpdk"
 141             infra = "-".join((tbed, nic, drv))
 142             lst_test = test.split("-")
 143             fsize = lst_test[0]
 144             core = lst_test[1] if lst_test[1] else "8C"
 145
 146             if tbs.get(dut, None) is None:
 147                 tbs[dut] = dict()
 148             if tbs[dut].get(dver, None) is None:
 149                 tbs[dut][dver] = dict()
 150             if tbs[dut][dver].get(infra, None) is None:
 151                 tbs[dut][dver][infra] = dict()
 152                 tbs[dut][dver][infra]["core"] = list()
 153                 tbs[dut][dver][infra]["fsize"] = list()
 154                 tbs[dut][dver][infra]["ttype"] = list()
 155             if core.upper() not in tbs[dut][dver][infra]["core"]:
 156                 tbs[dut][dver][infra]["core"].append(core.upper())
 157             if fsize.upper() not in tbs[dut][dver][infra]["fsize"]:
 158                 tbs[dut][dver][infra]["fsize"].append(fsize.upper())
 159             if row["test_type"] == "mrr":
 160                 if "MRR" not in tbs[dut][dver][infra]["ttype"]:
 161                     tbs[dut][dver][infra]["ttype"].append("MRR")
 162             elif row["test_type"] == "ndrpdr":
 163                 if "NDR" not in tbs[dut][dver][infra]["ttype"]:
 164                     tbs[dut][dver][infra]["ttype"].extend(
 165                         ("NDR", "PDR", "Latency")
 166                     )
 167             elif row["test_type"] == "hoststack" and \
 168                     row["tg_type"] in ("iperf", "vpp"):
 169                 if "BPS" not in tbs[dut][dver][infra]["ttype"]:
 170                     tbs[dut][dver][infra]["ttype"].append("BPS")
 171             elif row["test_type"] == "hoststack" and row["tg_type"] == "ab":
 172                 if "CPS" not in tbs[dut][dver][infra]["ttype"]:
 173                     tbs[dut][dver][infra]["ttype"].extend(("CPS", "RPS", ))
 174         self._tbs = tbs
 175
 176         # Read from files:
 177         self._html_layout = str()
 178         try:
 179             with open(self._html_layout_file, "r") as file_read:
 180                 self._html_layout = file_read.read()
 181         except IOError as err:
 182             raise RuntimeError(
 183                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 184             )
 185
 186         try:
 187             with open(self._graph_layout_file, "r") as file_read:
 188                 self._graph_layout = load(file_read, Loader=FullLoader)
 189         except IOError as err:
 190             raise RuntimeError(
 191                 f"Not possible to open the file {self._graph_layout_file}\n"
 192                 f"{err}"
 193             )
 194         except YAMLError as err:
 195             raise RuntimeError(
 196                 f"An error occurred while parsing the specification file "
 197                 f"{self._graph_layout_file}\n{err}"
 198             )
 199
 200         try:
 201             with open(self._tooltip_file, "r") as file_read:
 202                 self._tooltips = load(file_read, Loader=FullLoader)
 203         except IOError as err:
 204             logging.warning(
 205                 f"Not possible to open the file {self._tooltip_file}\n{err}"
 206             )
 207         except YAMLError as err:
 208             logging.warning(
 209                 f"An error occurred while parsing the specification file "
 210                 f"{self._tooltip_file}\n{err}"
 211             )
 212
 213         # Callbacks:
 214         if self._app is not None and hasattr(self, "callbacks"):
 215             self.callbacks(self._app)
 216
 217     @property
 218     def html_layout(self):
 219         return self._html_layout
 220
 221     def add_content(self):
 222         """Top level method which generated the web page.
 223
 224         It generates:
 225         - Store for user input data,
 226         - Navigation bar,
 227         - Main area with control panel and ploting area.
 228
 229         If no HTML layout is provided, an error message is displayed instead.
 230
 231         :returns: The HTML div with the whole page.
 232         :rtype: html.Div
 233         """
 234
 235         if self.html_layout and self._tbs:
 236             return html.Div(
 237                 id="div-main",
 238                 className="small",
 239                 children=[
 240                     dbc.Row(
 241                         id="row-navbar",
 242                         class_name="g-0",
 243                         children=[navbar_report((False, True, False, False)), ]
 244                     ),
 245                     dbc.Row(
 246                         id="row-main",
 247                         class_name="g-0",
 248                         children=[
 249                             dcc.Store(id="store-control-panel"),
 250                             dcc.Store(id="store-selected"),
 251                             dcc.Store(id="store-table-data"),
 252                             dcc.Store(id="store-filtered-table-data"),
 253                             dcc.Location(id="url", refresh=False),
 254                             self._add_ctrl_col(),
 255                             self._add_plotting_col()
 256                         ]
 257                     ),
 258                     dbc.Spinner(
 259                         dbc.Offcanvas(
 260                             class_name="w-75",
 261                             id="offcanvas-details",
 262                             title="Test Details",
 263                             placement="end",
 264                             is_open=False,
 265                             children=[]
 266                         ),
 267                         delay_show=C.SPINNER_DELAY
 268                     ),
 269                     dbc.Spinner(
 270                         dbc.Offcanvas(
 271                             class_name="w-50",
 272                             id="offcanvas-metadata",
 273                             title="Detailed Information",
 274                             placement="end",
 275                             is_open=False,
 276                             children=[
 277                                 dbc.Row(id="metadata-tput-lat"),
 278                                 dbc.Row(id="metadata-hdrh-graph")
 279                             ]
 280                         ),
 281                         delay_show=C.SPINNER_DELAY
 282                     ),
 283                     dbc.Offcanvas(
 284                         class_name="w-75",
 285                         id="offcanvas-documentation",
 286                         title="Documentation",
 287                         placement="end",
 288                         is_open=False,
 289                         children=html.Iframe(
 290                             src=C.URL_DOC_REL_NOTES,
 291                             width="100%",
 292                             height="100%"
 293                         )
 294                     )
 295                 ]
 296             )
 297         else:
 298             return html.Div(
 299                 id="div-main-error",
 300                 children=[
 301                     dbc.Alert(
 302                         [
 303                             "An Error Occured"
 304                         ],
 305                         color="danger"
 306                     )
 307                 ]
 308             )
 309
 310     def _add_ctrl_col(self) -> dbc.Col:
 311         """Add column with controls. It is placed on the left side.
 312
 313         :returns: Column with the control panel.
 314         :rtype: dbc.Col
 315         """
 316         return dbc.Col([
 317             html.Div(
 318                 children=self._add_ctrl_panel(),
 319                 className="sticky-top"
 320             )
 321         ])
 322
 323     def _add_plotting_col(self) -> dbc.Col:
 324         """Add column with plots. It is placed on the right side.
 325
 326         :returns: Column with plots.
 327         :rtype: dbc.Col
 328         """
 329         return dbc.Col(
 330             id="col-plotting-area",
 331             children=[
 332                 dbc.Spinner(
 333                     children=[
 334                         dbc.Row(
 335                             id="plotting-area",
 336                             class_name="g-0 p-0",
 337                             children=[
 338                                 C.PLACEHOLDER
 339                             ]
 340                         )
 341                     ]
 342                 )
 343             ],
 344             width=9
 345         )
 346
 347     def _add_ctrl_panel(self) -> list:
 348         """Add control panel.
 349
 350         :returns: Control panel.
 351         :rtype: list
 352         """
 353
 354         reference = [
 355             dbc.Row(
 356                 class_name="g-0 p-1",
 357                 children=[
 358                     dbc.InputGroup(
 359                         [
 360                             dbc.InputGroupText(
 361                                 show_tooltip(self._tooltips, "help-dut", "DUT")
 362                             ),
 363                             dbc.Select(
 364                                 id={"type": "ctrl-dd", "index": "dut"},
 365                                 placeholder="Select a Device under Test...",
 366                                 options=sorted(
 367                                     [
 368                                         {"label": k, "value": k} \
 369                                             for k in self._tbs.keys()
 370                                     ],
 371                                     key=lambda d: d["label"]
 372                                 )
 373                             )
 374                         ],
 375                         size="sm"
 376                     )
 377                 ]
 378             ),
 379             dbc.Row(
 380                 class_name="g-0 p-1",
 381                 children=[
 382                     dbc.InputGroup(
 383                         [
 384                             dbc.InputGroupText(show_tooltip(
 385                                 self._tooltips,
 386                                 "help-csit-dut",
 387                                 "CSIT and DUT Version"
 388                             )),
 389                             dbc.Select(
 390                                 id={"type": "ctrl-dd", "index": "dutver"},
 391                                 placeholder="Select a CSIT and DUT Version...")
 392                         ],
 393                         size="sm"
 394                     )
 395                 ]
 396             ),
 397             dbc.Row(
 398                 class_name="g-0 p-1",
 399                 children=[
 400                     dbc.InputGroup(
 401                         [
 402                             dbc.InputGroupText(show_tooltip(
 403                                 self._tooltips,
 404                                 "help-infra",
 405                                 "Infra"
 406                             )),
 407                             dbc.Select(
 408                                 id={"type": "ctrl-dd", "index": "infra"},
 409                                 placeholder=\
 410                                     "Select a Physical Test Bed Topology..."
 411                             )
 412                         ],
 413                         size="sm"
 414                     )
 415                 ]
 416             ),
 417             dbc.Row(
 418                 class_name="g-0 p-1",
 419                 children=[
 420                     dbc.InputGroup(
 421                         [
 422                             dbc.InputGroupText(show_tooltip(
 423                                 self._tooltips,
 424                                 "help-framesize",
 425                                 "Frame Size"
 426                             )),
 427                             dbc.Checklist(
 428                                 id={"type": "ctrl-cl", "index": "frmsize"},
 429                                 inline=True,
 430                                 class_name="ms-2"
 431                             )
 432                         ],
 433                         style={"align-items": "center"},
 434                         size="sm"
 435                     )
 436                 ]
 437             ),
 438             dbc.Row(
 439                 class_name="g-0 p-1",
 440                 children=[
 441                     dbc.InputGroup(
 442                         [
 443                             dbc.InputGroupText(show_tooltip(
 444                                 self._tooltips,
 445                                 "help-cores",
 446                                 "Number of Cores"
 447                             )),
 448                             dbc.Checklist(
 449                                 id={"type": "ctrl-cl", "index": "core"},
 450                                 inline=True,
 451                                 class_name="ms-2"
 452                             )
 453                         ],
 454                         style={"align-items": "center"},
 455                         size="sm"
 456                     )
 457                 ]
 458             ),
 459             dbc.Row(
 460                 class_name="g-0 p-1",
 461                 children=[
 462                     dbc.InputGroup(
 463                         [
 464                             dbc.InputGroupText(show_tooltip(
 465                                 self._tooltips,
 466                                 "help-measurement",
 467                                 "Measurement"
 468                             )),
 469                             dbc.Checklist(
 470                                 id={"type": "ctrl-cl", "index": "ttype"},
 471                                 inline=True,
 472                                 class_name="ms-2"
 473                             )
 474                         ],
 475                         style={"align-items": "center"},
 476                         size="sm"
 477                     )
 478                 ]
 479             )
 480         ]
 481
 482         compare = [
 483             dbc.Row(
 484                 class_name="g-0 p-1",
 485                 children=[
 486                     dbc.InputGroup(
 487                         [
 488                             dbc.InputGroupText(show_tooltip(
 489                                 self._tooltips,
 490                                 "help-cmp-parameter",
 491                                 "Parameter"
 492                             )),
 493                             dbc.Select(
 494                                 id={"type": "ctrl-dd", "index": "cmpprm"},
 495                                 placeholder="Select a Parameter..."
 496                             )
 497                         ],
 498                         size="sm"
 499                     )
 500                 ]
 501             ),
 502             dbc.Row(
 503                 class_name="g-0 p-1",
 504                 children=[
 505                     dbc.InputGroup(
 506                         [
 507                             dbc.InputGroupText(show_tooltip(
 508                                 self._tooltips,
 509                                 "help-cmp-value",
 510                                 "Value"
 511                             )),
 512                             dbc.Select(
 513                                 id={"type": "ctrl-dd", "index": "cmpval"},
 514                                 placeholder="Select a Value..."
 515                             )
 516                         ],
 517                         size="sm"
 518                     )
 519                 ]
 520             )
 521         ]
 522
 523         processing = [
 524             dbc.Row(
 525                 class_name="g-0 p-1",
 526                 children=[
 527                     dbc.InputGroup(
 528                         children = [
 529                             dbc.Checklist(
 530                                 id="normalize",
 531                                 options=[{
 532                                     "value": "normalize",
 533                                     "label": "Normalize to 2GHz CPU frequency"
 534                                 }],
 535                                 value=[],
 536                                 inline=True,
 537                                 class_name="ms-2"
 538                             ),
 539                             dbc.Checklist(
 540                                 id="outliers",
 541                                 options=[{
 542                                     "value": "outliers",
 543                                     "label": "Remove Extreme Outliers"
 544                                 }],
 545                                 value=[],
 546                                 inline=True,
 547                                 class_name="ms-2"
 548                             )
 549                         ],
 550                         style={"align-items": "center"},
 551                         size="sm"
 552                     )
 553                 ]
 554             )
 555         ]
 556
 557         return [
 558             dbc.Row(
 559                 dbc.Card(
 560                     [
 561                         dbc.CardHeader(
 562                             html.H5("Reference Value")
 563                         ),
 564                         dbc.CardBody(
 565                             children=reference,
 566                             class_name="g-0 p-0"
 567                         )
 568                     ],
 569                     color="secondary",
 570                     outline=True
 571                 ),
 572                 class_name="g-0 p-1"
 573             ),
 574             dbc.Row(
 575                 dbc.Card(
 576                     [
 577                         dbc.CardHeader(
 578                             html.H5("Compared Value")
 579                         ),
 580                         dbc.CardBody(
 581                             children=compare,
 582                             class_name="g-0 p-0"
 583                         )
 584                     ],
 585                     color="secondary",
 586                     outline=True
 587                 ),
 588                 class_name="g-0 p-1"
 589             ),
 590             dbc.Row(
 591                 dbc.Card(
 592                     [
 593                         dbc.CardHeader(
 594                             html.H5("Data Manipulations")
 595                         ),
 596                         dbc.CardBody(
 597                             children=processing,
 598                             class_name="g-0 p-0"
 599                         )
 600                     ],
 601                     color="secondary",
 602                     outline=True
 603                 ),
 604                 class_name="g-0 p-1"
 605             )
 606         ]
 607
 608     @staticmethod
 609     def _get_plotting_area(
 610             title: str,
 611             table: pd.DataFrame,
 612             url: str
 613         ) -> list:
 614         """Generate the plotting area with all its content.
 615
 616         :param title: The title of the comparison table.
 617         :param table: Comparison table to be displayed.
 618         :param url: URL to be displayed in the modal window.
 619         :type title: str
 620         :type table: pandas.DataFrame
 621         :type url: str
 622         :returns: List of rows with elements to be displayed in the plotting
 623             area.
 624         :rtype: list
 625         """
 626
 627         if table.empty:
 628             return dbc.Row(
 629                 dbc.Col(
 630                     children=dbc.Alert(
 631                         "No data for comparison.",
 632                         color="danger"
 633                     ),
 634                     class_name="g-0 p-1",
 635                 ),
 636                 class_name="g-0 p-0"
 637             )
 638
 639         cols = list()
 640         for idx, col in enumerate(table.columns):
 641             if idx == 0:
 642                 cols.append({
 643                     "name": ["", col],
 644                     "id": col,
 645                     "deletable": False,
 646                     "selectable": False,
 647                     "type": "text"
 648                 })
 649             else:
 650                 l_col = col.rsplit(" ", 2)
 651                 cols.append({
 652                     "name": [l_col[0], " ".join(l_col[-2:])],
 653                     "id": col,
 654                     "deletable": False,
 655                     "selectable": False,
 656                     "type": "numeric",
 657                     "format": Format(precision=2, scheme=Scheme.fixed)
 658                 })
 659
 660         return [
 661             dbc.Row(
 662                 children=html.H5(title),
 663                 class_name="g-0 p-1"
 664             ),
 665             dbc.Row(
 666                 children=[
 667                     dbc.Col(
 668                         children=dash_table.DataTable(
 669                             id={"type": "table", "index": "comparison"},
 670                             columns=cols,
 671                             data=table.to_dict("records"),
 672                             merge_duplicate_headers=True,
 673                             editable=False,
 674                             filter_action="custom",
 675                             filter_query="",
 676                             sort_action="custom",
 677                             sort_mode="multi",
 678                             selected_columns=[],
 679                             selected_rows=[],
 680                             page_action="none",
 681                             style_cell={"textAlign": "right"},
 682                             style_cell_conditional=[{
 683                                 "if": {"column_id": "Test Name"},
 684                                 "textAlign": "left"
 685                             }]
 686                         ),
 687                         class_name="g-0 p-1"
 688                     )
 689                 ],
 690                 class_name="g-0 p-0"
 691             ),
 692             dbc.Row(
 693                 [
 694                     dbc.Col([html.Div(
 695                         [
 696                             dbc.Button(
 697                                 id="plot-btn-url",
 698                                 children="Show URL",
 699                                 class_name="me-1",
 700                                 color="info",
 701                                 style={
 702                                     "text-transform": "none",
 703                                     "padding": "0rem 1rem"
 704                                 }
 705                             ),
 706                             dbc.Modal(
 707                                 [
 708                                     dbc.ModalHeader(dbc.ModalTitle("URL")),
 709                                     dbc.ModalBody(url)
 710                                 ],
 711                                 id="plot-mod-url",
 712                                 size="xl",
 713                                 is_open=False,
 714                                 scrollable=True
 715                             ),
 716                             dbc.Button(
 717                                 id="plot-btn-download",
 718                                 children="Download Table",
 719                                 class_name="me-1",
 720                                 color="info",
 721                                 style={
 722                                     "text-transform": "none",
 723                                     "padding": "0rem 1rem"
 724                                 }
 725                             ),
 726                             dcc.Download(id="download-iterative-data"),
 727                             dbc.Button(
 728                                 id="plot-btn-download-raw",
 729                                 children="Download Raw Data",
 730                                 class_name="me-1",
 731                                 color="info",
 732                                 style={
 733                                     "text-transform": "none",
 734                                     "padding": "0rem 1rem"
 735                                 }
 736                             ),
 737                             dcc.Download(id="download-raw-data")
 738                         ],
 739                         className=\
 740                             "d-grid gap-0 d-md-flex justify-content-md-end"
 741                     )])
 742                 ],
 743                 class_name="g-0 p-0"
 744             ),
 745             dbc.Row(
 746                 children=C.PLACEHOLDER,
 747                 class_name="g-0 p-1"
 748             )
 749         ]
 750
 751     def callbacks(self, app):
 752         """Callbacks for the whole application.
 753
 754         :param app: The application.
 755         :type app: Flask
 756         """
 757
 758         @app.callback(
 759             [
 760                 Output("store-control-panel", "data"),
 761                 Output("store-selected", "data"),
 762                 Output("store-table-data", "data"),
 763                 Output("store-filtered-table-data", "data"),
 764                 Output("plotting-area", "children"),
 765                 Output({"type": "table", "index": ALL}, "data"),
 766                 Output({"type": "ctrl-dd", "index": "dut"}, "value"),
 767                 Output({"type": "ctrl-dd", "index": "dutver"}, "options"),
 768                 Output({"type": "ctrl-dd", "index": "dutver"}, "disabled"),
 769                 Output({"type": "ctrl-dd", "index": "dutver"}, "value"),
 770                 Output({"type": "ctrl-dd", "index": "infra"}, "options"),
 771                 Output({"type": "ctrl-dd", "index": "infra"}, "disabled"),
 772                 Output({"type": "ctrl-dd", "index": "infra"}, "value"),
 773                 Output({"type": "ctrl-cl", "index": "core"}, "options"),
 774                 Output({"type": "ctrl-cl", "index": "core"}, "value"),
 775                 Output({"type": "ctrl-cl", "index": "frmsize"}, "options"),
 776                 Output({"type": "ctrl-cl", "index": "frmsize"}, "value"),
 777                 Output({"type": "ctrl-cl", "index": "ttype"}, "options"),
 778                 Output({"type": "ctrl-cl", "index": "ttype"}, "value"),
 779                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "options"),
 780                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "disabled"),
 781                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "value"),
 782                 Output({"type": "ctrl-dd", "index": "cmpval"}, "options"),
 783                 Output({"type": "ctrl-dd", "index": "cmpval"}, "disabled"),
 784                 Output({"type": "ctrl-dd", "index": "cmpval"}, "value"),
 785                 Output("normalize", "value"),
 786                 Output("outliers", "value")
 787             ],
 788             [
 789                 State("store-control-panel", "data"),
 790                 State("store-selected", "data"),
 791                 State("store-table-data", "data"),
 792                 State("store-filtered-table-data", "data"),
 793                 State({"type": "table", "index": ALL}, "data")
 794             ],
 795             [
 796                 Input("url", "href"),
 797                 Input("normalize", "value"),
 798                 Input("outliers", "value"),
 799                 Input({"type": "table", "index": ALL}, "filter_query"),
 800                 Input({"type": "table", "index": ALL}, "sort_by"),
 801                 Input({"type": "ctrl-dd", "index": ALL}, "value"),
 802                 Input({"type": "ctrl-cl", "index": ALL}, "value"),
 803                 Input({"type": "ctrl-btn", "index": ALL}, "n_clicks")
 804             ]
 805         )
 806         def _update_application(
 807                 control_panel: dict,
 808                 selected: dict,
 809                 store_table_data: list,
 810                 filtered_data: list,
 811                 table_data: list,
 812                 href: str,
 813                 normalize: list,
 814                 outliers: bool,
 815                 *_
 816             ) -> tuple:
 817             """Update the application when the event is detected.
 818             """
 819
 820             ctrl_panel = ControlPanel(CP_PARAMS, control_panel)
 821
 822             if selected is None:
 823                 selected = {
 824                     "reference": {
 825                         "set": False,
 826                     },
 827                     "compare": {
 828                         "set": False,
 829                     }
 830                 }
 831
 832             # Parse the url:
 833             parsed_url = url_decode(href)
 834             if parsed_url:
 835                 url_params = parsed_url["params"]
 836             else:
 837                 url_params = None
 838
 839             on_draw = False
 840             plotting_area = no_update
 841
 842             trigger = Trigger(callback_context.triggered)
 843             if trigger.type == "url" and url_params:
 844                 process_url = False
 845                 try:
 846                     selected = literal_eval(url_params["selected"][0])
 847                     r_sel = selected["reference"]["selection"]
 848                     c_sel = selected["compare"]
 849                     normalize = literal_eval(url_params["norm"][0])
 850                     try:  # Necessary for backward compatibility
 851                         outliers = literal_eval(url_params["outliers"][0])
 852                     except (KeyError, IndexError, AttributeError):
 853                         outliers = list()
 854                     process_url = bool(
 855                         (selected["reference"]["set"] == True) and
 856                         (c_sel["set"] == True)
 857                     )
 858                 except (KeyError, IndexError, AttributeError):
 859                     pass
 860                 if process_url:
 861                     ctrl_panel.set({
 862                         "dut-val": r_sel["dut"],
 863                         "dutver-opt": generate_options(
 864                             self._tbs[r_sel["dut"]].keys()
 865                         ),
 866                         "dutver-dis": False,
 867                         "dutver-val": r_sel["dutver"],
 868                         "infra-opt": generate_options(
 869                             self._tbs[r_sel["dut"]][r_sel["dutver"]].keys()
 870                         ),
 871                         "infra-dis": False,
 872                         "infra-val": r_sel["infra"],
 873                         "core-opt": generate_options(
 874                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 875                                 [r_sel["infra"]]["core"]
 876                         ),
 877                         "core-val": r_sel["core"],
 878                         "frmsize-opt": generate_options(
 879                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 880                                 [r_sel["infra"]]["fsize"]
 881                         ),
 882                         "frmsize-val": r_sel["frmsize"],
 883                         "ttype-opt": generate_options(
 884                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 885                                 [r_sel["infra"]]["ttype"]
 886                         ),
 887                         "ttype-val": r_sel["ttype"],
 888                         "normalize-val": normalize,
 889                         "outliers-val": outliers
 890                     })
 891                     opts = list()
 892                     for itm, label in CMP_PARAMS.items():
 893                         if len(ctrl_panel.get(f"{itm}-opt")) > 1:
 894                             opts.append({"label": label, "value": itm})
 895                     ctrl_panel.set({
 896                         "cmp-par-opt": opts,
 897                         "cmp-par-dis": False,
 898                         "cmp-par-val": c_sel["parameter"]
 899                     })
 900                     opts = list()
 901                     for itm in ctrl_panel.get(f"{c_sel['parameter']}-opt"):
 902                         set_val = ctrl_panel.get(f"{c_sel['parameter']}-val")
 903                         if isinstance(set_val, list):
 904                             if itm["value"] not in set_val:
 905                                 opts.append(itm)
 906                         else:
 907                             if itm["value"] != set_val:
 908                                 opts.append(itm)
 909                     ctrl_panel.set({
 910                         "cmp-val-opt": opts,
 911                         "cmp-val-dis": False,
 912                         "cmp-val-val": c_sel["value"]
 913                     })
 914                     on_draw = True
 915             elif trigger.type == "normalize":
 916                 ctrl_panel.set({"normalize-val": normalize})
 917                 on_draw = True
 918             elif trigger.type == "outliers":
 919                 ctrl_panel.set({"outliers-val": outliers})
 920                 on_draw = True
 921             elif trigger.type == "ctrl-dd":
 922                 if trigger.idx == "dut":
 923                     try:
 924                         opts = generate_options(self._tbs[trigger.value].keys())
 925                         disabled = False
 926                     except KeyError:
 927                         opts = list()
 928                         disabled = True
 929                     ctrl_panel.set({
 930                         "dut-val": trigger.value,
 931                         "dutver-opt": opts,
 932                         "dutver-dis": disabled,
 933                         "dutver-val": str(),
 934                         "infra-opt": list(),
 935                         "infra-dis": True,
 936                         "infra-val": str(),
 937                         "core-opt": list(),
 938                         "core-val": list(),
 939                         "frmsize-opt": list(),
 940                         "frmsize-val": list(),
 941                         "ttype-opt": list(),
 942                         "ttype-val": list(),
 943                         "cmp-par-opt": list(),
 944                         "cmp-par-dis": True,
 945                         "cmp-par-val": str(),
 946                         "cmp-val-opt": list(),
 947                         "cmp-val-dis": True,
 948                         "cmp-val-val": str()
 949                     })
 950                 elif trigger.idx == "dutver":
 951                     try:
 952                         dut = ctrl_panel.get("dut-val")
 953                         dver = self._tbs[dut][trigger.value]
 954                         opts = generate_options(dver.keys())
 955                         disabled = False
 956                     except KeyError:
 957                         opts = list()
 958                         disabled = True
 959                     ctrl_panel.set({
 960                         "dutver-val": trigger.value,
 961                         "infra-opt": opts,
 962                         "infra-dis": disabled,
 963                         "infra-val": str(),
 964                         "core-opt": list(),
 965                         "core-val": list(),
 966                         "frmsize-opt": list(),
 967                         "frmsize-val": list(),
 968                         "ttype-opt": list(),
 969                         "ttype-val": list(),
 970                         "cmp-par-opt": list(),
 971                         "cmp-par-dis": True,
 972                         "cmp-par-val": str(),
 973                         "cmp-val-opt": list(),
 974                         "cmp-val-dis": True,
 975                         "cmp-val-val": str()
 976                     })
 977                 elif trigger.idx == "infra":
 978                     dut = ctrl_panel.get("dut-val")
 979                     dver = ctrl_panel.get("dutver-val")
 980                     if all((dut, dver, trigger.value, )):
 981                         driver = self._tbs[dut][dver][trigger.value]
 982                         ctrl_panel.set({
 983                             "infra-val": trigger.value,
 984                             "core-opt": generate_options(driver["core"]),
 985                             "core-val": list(),
 986                             "frmsize-opt": generate_options(driver["fsize"]),
 987                             "frmsize-val": list(),
 988                             "ttype-opt": generate_options(driver["ttype"]),
 989                             "ttype-val": list(),
 990                             "cmp-par-opt": list(),
 991                             "cmp-par-dis": True,
 992                             "cmp-par-val": str(),
 993                             "cmp-val-opt": list(),
 994                             "cmp-val-dis": True,
 995                             "cmp-val-val": str()
 996                         })
 997                 elif trigger.idx == "cmpprm":
 998                     value = trigger.value
 999                     opts = list()
1000                     for itm in ctrl_panel.get(f"{value}-opt"):
1001                         set_val = ctrl_panel.get(f"{value}-val")
1002                         if isinstance(set_val, list):
1003                             if itm["value"] == "Latency":
1004                                 continue
1005                             if itm["value"] not in set_val:
1006                                 opts.append(itm)
1007                         else:
1008                             if itm["value"] != set_val:
1009                                 opts.append(itm)
1010                     ctrl_panel.set({
1011                         "cmp-par-val": value,
1012                         "cmp-val-opt": opts,
1013                         "cmp-val-dis": False,
1014                         "cmp-val-val": str()
1015                     })
1016                 elif trigger.idx == "cmpval":
1017                     ctrl_panel.set({"cmp-val-val": trigger.value})
1018                     selected["reference"] = {
1019                         "set": True,
1020                         "selection": {
1021                             "dut": ctrl_panel.get("dut-val"),
1022                             "dutver": ctrl_panel.get("dutver-val"),
1023                             "infra": ctrl_panel.get("infra-val"),
1024                             "core": ctrl_panel.get("core-val"),
1025                             "frmsize": ctrl_panel.get("frmsize-val"),
1026                             "ttype": ctrl_panel.get("ttype-val")
1027                         }
1028                     }
1029                     selected["compare"] = {
1030                         "set": True,
1031                         "parameter": ctrl_panel.get("cmp-par-val"),
1032                         "value": trigger.value
1033                     }
1034                     on_draw = True
1035             elif trigger.type == "ctrl-cl":
1036                 ctrl_panel.set({f"{trigger.idx}-val": trigger.value})
1037                 if all((ctrl_panel.get("core-val"),
1038                         ctrl_panel.get("frmsize-val"),
1039                         ctrl_panel.get("ttype-val"), )):
1040                     if "Latency" in ctrl_panel.get("ttype-val"):
1041                         ctrl_panel.set({"ttype-val": ["Latency", ]})
1042                     opts = list()
1043                     for itm, label in CMP_PARAMS.items():
1044                         if "Latency" in ctrl_panel.get("ttype-val") and \
1045                                 itm == "ttype":
1046                             continue
1047                         if len(ctrl_panel.get(f"{itm}-opt")) > 1:
1048                             if isinstance(ctrl_panel.get(f"{itm}-val"), list):
1049                                 if len(ctrl_panel.get(f"{itm}-opt")) == \
1050                                         len(ctrl_panel.get(f"{itm}-val")):
1051                                     continue
1052                             opts.append({"label": label, "value": itm})
1053                     ctrl_panel.set({
1054                         "cmp-par-opt": opts,
1055                         "cmp-par-dis": False,
1056                         "cmp-par-val": str(),
1057                         "cmp-val-opt": list(),
1058                         "cmp-val-dis": True,
1059                         "cmp-val-val": str()
1060                     })
1061                 else:
1062                     ctrl_panel.set({
1063                         "cmp-par-opt": list(),
1064                         "cmp-par-dis": True,
1065                         "cmp-par-val": str(),
1066                         "cmp-val-opt": list(),
1067                         "cmp-val-dis": True,
1068                         "cmp-val-val": str()
1069                     })
1070             elif trigger.type == "table" and trigger.idx == "comparison":
1071                 if trigger.parameter == "filter_query":
1072                     filtered_data = filter_table_data(
1073                         store_table_data,
1074                         trigger.value
1075                     )
1076                 elif trigger.parameter == "sort_by":
1077                     filtered_data = sort_table_data(
1078                         store_table_data,
1079                         trigger.value
1080                     )
1081                 table_data = [filtered_data, ]
1082
1083             if all((on_draw, selected["reference"]["set"],
1084                     selected["compare"]["set"], )):
1085                 title, table = comparison_table(
1086                     data=self._data,
1087                     selected=selected,
1088                     normalize=normalize,
1089                     format="html",
1090                     remove_outliers=outliers
1091                 )
1092                 plotting_area = self._get_plotting_area(
1093                     title=title,
1094                     table=table,
1095                     url=gen_new_url(
1096                         parsed_url,
1097                         params={
1098                             "selected": selected,
1099                             "norm": normalize,
1100                             "outliers": outliers
1101                         }
1102                     )
1103                 )
1104                 store_table_data = table.to_dict("records")
1105                 filtered_data = store_table_data
1106                 if table_data:
1107                     table_data = [store_table_data, ]
1108
1109             ret_val = [
1110                 ctrl_panel.panel,
1111                 selected,
1112                 store_table_data,
1113                 filtered_data,
1114                 plotting_area,
1115                 table_data
1116             ]
1117             ret_val.extend(ctrl_panel.values)
1118             return ret_val
1119
1120         @app.callback(
1121             Output("plot-mod-url", "is_open"),
1122             Input("plot-btn-url", "n_clicks"),
1123             State("plot-mod-url", "is_open")
1124         )
1125         def toggle_plot_mod_url(n, is_open):
1126             """Toggle the modal window with url.
1127             """
1128             if n:
1129                 return not is_open
1130             return is_open
1131
1132         @app.callback(
1133             Output("download-iterative-data", "data"),
1134             State("store-table-data", "data"),
1135             State("store-filtered-table-data", "data"),
1136             Input("plot-btn-download", "n_clicks"),
1137             prevent_initial_call=True
1138         )
1139         def _download_comparison_data(
1140                 table_data: list,
1141                 filtered_table_data: list,
1142                 _: int
1143             ) -> dict:
1144             """Download the data.
1145
1146             :param table_data: Original unfiltered table data.
1147             :param filtered_table_data: Filtered table data.
1148             :type table_data: list
1149             :type filtered_table_data: list
1150             :returns: dict of data frame content (base64 encoded) and meta data
1151                 used by the Download component.
1152             :rtype: dict
1153             """
1154
1155             if not table_data:
1156                 raise PreventUpdate
1157
1158             if filtered_table_data:
1159                 table = pd.DataFrame.from_records(filtered_table_data)
1160             else:
1161                 table = pd.DataFrame.from_records(table_data)
1162
1163             return dcc.send_data_frame(table.to_csv, C.COMP_DOWNLOAD_FILE_NAME)
1164
1165         @app.callback(
1166             Output("download-raw-data", "data"),
1167             State("store-selected", "data"),
1168             Input("plot-btn-download-raw", "n_clicks"),
1169             prevent_initial_call=True
1170         )
1171         def _download_raw_comparison_data(selected: dict, _: int) -> dict:
1172             """Download the data.
1173
1174             :param selected: Selected tests.
1175             :type selected: dict
1176             :returns: dict of data frame content (base64 encoded) and meta data
1177                 used by the Download component.
1178             :rtype: dict
1179             """
1180
1181             if not selected:
1182                 raise PreventUpdate
1183
1184             _, table = comparison_table(
1185                     data=self._data,
1186                     selected=selected,
1187                     normalize=False,
1188                     remove_outliers=False,
1189                     raw_data=True
1190                 )
1191
1192             return dcc.send_data_frame(
1193                 table.dropna(how="all", axis=1).to_csv,
1194                 f"raw_{C.COMP_DOWNLOAD_FILE_NAME}"
1195             )
1196
1197         @app.callback(
1198             Output("offcanvas-documentation", "is_open"),
1199             Input("btn-documentation", "n_clicks"),
1200             State("offcanvas-documentation", "is_open")
1201         )
1202         def toggle_offcanvas_documentation(n_clicks, is_open):
1203             if n_clicks:
1204                 return not is_open
1205             return is_open
1206
1207         @app.callback(
1208             Output("offcanvas-details", "is_open"),
1209             Output("offcanvas-details", "children"),
1210             State("store-selected", "data"),
1211             State("store-filtered-table-data", "data"),
1212             State("normalize", "value"),
1213             State("outliers", "value"),
1214             Input({"type": "table", "index": ALL}, "active_cell"),
1215             prevent_initial_call=True
1216         )
1217         def show_test_data(cp_sel, table, normalize, outliers, *_):
1218             """Show offcanvas with graphs and tables based on selected test(s).
1219             """
1220
1221             trigger = Trigger(callback_context.triggered)
1222             if not all((trigger.value, cp_sel["reference"]["set"], \
1223                         cp_sel["compare"]["set"])):
1224                 raise PreventUpdate
1225
1226             try:
1227                 test_name = pd.DataFrame.from_records(table).\
1228                     iloc[[trigger.value["row"]]]["Test Name"].iloc[0]
1229                 dut = cp_sel["reference"]["selection"]["dut"]
1230                 rls, dutver = cp_sel["reference"]["selection"]["dutver"].\
1231                     split("-", 1)
1232                 phy = cp_sel["reference"]["selection"]["infra"]
1233                 framesize, core, test_id = test_name.split("-", 2)
1234                 test, ttype = test_id.rsplit("-", 1)
1235                 ttype = "pdr" if ttype == "latency" else ttype
1236                 l_phy = phy.split("-")
1237                 tb = "-".join(l_phy[:2])
1238                 nic = l_phy[2]
1239                 stype = "ndrpdr" if ttype in ("ndr", "pdr") else ttype
1240             except(KeyError, IndexError, AttributeError, ValueError):
1241                 raise PreventUpdate
1242
1243             df = pd.DataFrame(self._data.loc[(
1244                     (self._data["dut_type"] == dut) &
1245                     (self._data["dut_version"] == dutver) &
1246                     (self._data["release"] == rls)
1247                 )])
1248             df = df[df.job.str.endswith(tb)]
1249             df = df[df.test_id.str.contains(
1250                 f"{nic}.*{test}-{stype}", regex=True
1251             )]
1252             if df.empty:
1253                 raise PreventUpdate
1254
1255             l_test_id = df["test_id"].iloc[0].split(".")
1256             area = ".".join(l_test_id[3:-2])
1257
1258             r_sel = {
1259                 "id": f"{test}-{ttype}",
1260                 "rls": rls,
1261                 "dut": dut,
1262                 "dutver": dutver,
1263                 "phy": phy,
1264                 "area": area,
1265                 "test": test,
1266                 "framesize": framesize,
1267                 "core": core,
1268                 "testtype": ttype
1269             }
1270
1271             c_sel = deepcopy(r_sel)
1272             param = cp_sel["compare"]["parameter"]
1273             val = cp_sel["compare"]["value"].lower()
1274             if param == "dutver":
1275                 c_sel["rls"], c_sel["dutver"] = val.split("-", 1)
1276             elif param == "ttype":
1277                 c_sel["id"] = f"{test}-{val}"
1278                 c_sel["testtype"] = val
1279             elif param == "infra":
1280                 c_sel["phy"] = val
1281             else:
1282                 c_sel[param] = val
1283
1284             r_sel["id"] = "-".join(
1285                 (r_sel["phy"], r_sel["framesize"], r_sel["core"], r_sel["id"])
1286             )
1287             c_sel["id"] = "-".join(
1288                 (c_sel["phy"], c_sel["framesize"], c_sel["core"], c_sel["id"])
1289             )
1290             selected = [r_sel, c_sel]
1291
1292             indexes = ("tput", "bandwidth", "lat")
1293             graphs = graph_iterative(
1294                 self._data,
1295                 selected,
1296                 self._graph_layout,
1297                 bool(normalize),
1298                 bool(outliers)
1299             )
1300             cols = list()
1301             for graph, idx in zip(graphs, indexes):
1302                 if graph:
1303                     cols.append(dbc.Col(dcc.Graph(
1304                         figure=graph,
1305                         id={"type": "graph-iter", "index": idx},
1306                     )))
1307             if not cols:
1308                 cols="No data."
1309             ret_val = [
1310                 dbc.Row(
1311                     class_name="g-0 p-0",
1312                     children=dbc.Alert(test, color="info"),
1313                 ),
1314                 dbc.Row(class_name="g-0 p-0", children=cols)
1315             ]
1316
1317             return True, ret_val
1318
1319         @app.callback(
1320             Output("metadata-tput-lat", "children"),
1321             Output("metadata-hdrh-graph", "children"),
1322             Output("offcanvas-metadata", "is_open"),
1323             Input({"type": "graph-iter", "index": ALL}, "clickData"),
1324             prevent_initial_call=True
1325         )
1326         def _show_metadata_from_graph(iter_data: dict) -> tuple:
1327             """Generates the data for the offcanvas displayed when a particular
1328             point in a graph is clicked on.
1329             """
1330
1331             trigger = Trigger(callback_context.triggered)
1332             if not trigger.value:
1333                 raise PreventUpdate
1334
1335             if trigger.type == "graph-iter":
1336                 return show_iterative_graph_data(
1337                     trigger, iter_data, self._graph_layout)
1338             else:
1339                 raise PreventUpdate