csit.infra.dash/app/cdash/comparisons/layout.py

   1 # Copyright (c) 2024 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17
  18 import logging
  19 import pandas as pd
  20 import dash_bootstrap_components as dbc
  21
  22 from flask import Flask
  23 from dash import dcc, html, dash_table, callback_context, no_update, ALL
  24 from dash import Input, Output, State
  25 from dash.exceptions import PreventUpdate
  26 from dash.dash_table.Format import Format, Scheme
  27 from ast import literal_eval
  28 from yaml import load, FullLoader, YAMLError
  29 from copy import deepcopy
  30
  31 from ..utils.constants import Constants as C
  32 from ..utils.control_panel import ControlPanel
  33 from ..utils.trigger import Trigger
  34 from ..utils.url_processing import url_decode
  35 from ..utils.utils import generate_options, gen_new_url, navbar_report, \
  36     filter_table_data, sort_table_data, show_iterative_graph_data, show_tooltip
  37 from .tables import comparison_table
  38 from ..report.graphs import graph_iterative
  39
  40
  41 # Control panel partameters and their default values.
  42 CP_PARAMS = {
  43     "dut-val": str(),
  44     "dutver-opt": list(),
  45     "dutver-dis": True,
  46     "dutver-val": str(),
  47     "infra-opt": list(),
  48     "infra-dis": True,
  49     "infra-val": str(),
  50     "core-opt": list(),
  51     "core-val": list(),
  52     "frmsize-opt": list(),
  53     "frmsize-val": list(),
  54     "ttype-opt": list(),
  55     "ttype-val": list(),
  56     "cmp-par-opt": list(),
  57     "cmp-par-dis": True,
  58     "cmp-par-val": str(),
  59     "cmp-val-opt": list(),
  60     "cmp-val-dis": True,
  61     "cmp-val-val": str(),
  62     "normalize-val": list(),
  63     "outliers-val": list()
  64 }
  65
  66 # List of comparable parameters.
  67 CMP_PARAMS = {
  68     "dutver": "Release and Version",
  69     "infra": "Infrastructure",
  70     "frmsize": "Frame Size",
  71     "core": "Number of Cores",
  72     "ttype": "Measurement"
  73 }
  74
  75
  76 class Layout:
  77     """The layout of the dash app and the callbacks.
  78     """
  79
  80     def __init__(
  81             self,
  82             app: Flask,
  83             data_iterative: pd.DataFrame,
  84             html_layout_file: str,
  85             graph_layout_file: str,
  86             tooltip_file: str
  87         ) -> None:
  88         """Initialization:
  89         - save the input parameters,
  90         - prepare data for the control panel,
  91         - read HTML layout file,
  92         - read graph layout file,
  93         - read tooltips from the tooltip file.
  94
  95         :param app: Flask application running the dash application.
  96         :param data_iterative: Iterative data to be used in comparison tables.
  97         :param html_layout_file: Path and name of the file specifying the HTML
  98             layout of the dash application.
  99         :param tooltip_file: Path and name of the yaml file specifying the
 100             tooltips.
 101         :param graph_layout_file: Path and name of the file with layout of
 102             plot.ly graphs.
 103         :type app: Flask
 104         :type data_iterative: pandas.DataFrame
 105         :type html_layout_file: str
 106         :type graph_layout_file: str
 107         :type tooltip_file: str
 108         """
 109
 110         # Inputs
 111         self._app = app
 112         self._data = data_iterative
 113         self._html_layout_file = html_layout_file
 114         self._graph_layout_file = graph_layout_file
 115         self._tooltip_file = tooltip_file
 116
 117         # Get structure of tests:
 118         tbs = dict()
 119         cols = [
 120             "job", "test_id", "test_type", "dut_type", "dut_version", "tg_type",
 121             "release", "passed"
 122         ]
 123         for _, row in self._data[cols].drop_duplicates().iterrows():
 124             lst_job = row["job"].split("-")
 125             dut = lst_job[1]
 126             dver = f"{row['release']}-{row['dut_version']}"
 127             tbed = "-".join(lst_job[-2:])
 128             lst_test_id = row["test_id"].split(".")
 129
 130             suite = lst_test_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
 131                 replace("2n-", "")
 132             test = lst_test_id[-1]
 133             nic = suite.split("-")[0]
 134             for driver in C.DRIVERS:
 135                 if driver in test:
 136                     drv = driver.replace("-", "_")
 137                     test = test.replace(f"{driver}-", "")
 138                     break
 139             else:
 140                 drv = "dpdk"
 141             infra = "-".join((tbed, nic, drv))
 142             lst_test = test.split("-")
 143             fsize = lst_test[0]
 144             core = lst_test[1] if lst_test[1] else "8C"
 145
 146             if tbs.get(dut, None) is None:
 147                 tbs[dut] = dict()
 148             if tbs[dut].get(dver, None) is None:
 149                 tbs[dut][dver] = dict()
 150             if tbs[dut][dver].get(infra, None) is None:
 151                 tbs[dut][dver][infra] = dict()
 152                 tbs[dut][dver][infra]["core"] = list()
 153                 tbs[dut][dver][infra]["fsize"] = list()
 154                 tbs[dut][dver][infra]["ttype"] = list()
 155             if core.upper() not in tbs[dut][dver][infra]["core"]:
 156                 tbs[dut][dver][infra]["core"].append(core.upper())
 157             if fsize.upper() not in tbs[dut][dver][infra]["fsize"]:
 158                 tbs[dut][dver][infra]["fsize"].append(fsize.upper())
 159             if row["test_type"] == "mrr":
 160                 if "MRR" not in tbs[dut][dver][infra]["ttype"]:
 161                     tbs[dut][dver][infra]["ttype"].append("MRR")
 162             elif row["test_type"] == "ndrpdr":
 163                 if "NDR" not in tbs[dut][dver][infra]["ttype"]:
 164                     tbs[dut][dver][infra]["ttype"].extend(
 165                         ("NDR", "PDR", "Latency")
 166                     )
 167             elif row["test_type"] == "hoststack" and \
 168                     row["tg_type"] in ("iperf", "vpp"):
 169                 if "BPS" not in tbs[dut][dver][infra]["ttype"]:
 170                     tbs[dut][dver][infra]["ttype"].append("BPS")
 171             elif row["test_type"] == "hoststack" and row["tg_type"] == "ab":
 172                 if "CPS" not in tbs[dut][dver][infra]["ttype"]:
 173                     tbs[dut][dver][infra]["ttype"].extend(("CPS", "RPS", ))
 174             elif row["test_type"] == "soak":
 175                 if "SOAK" not in tbs[dut][dver][infra]["ttype"]:
 176                     tbs[dut][dver][infra]["ttype"].append("SOAK")
 177         self._tbs = tbs
 178
 179         # Read from files:
 180         self._html_layout = str()
 181         try:
 182             with open(self._html_layout_file, "r") as file_read:
 183                 self._html_layout = file_read.read()
 184         except IOError as err:
 185             raise RuntimeError(
 186                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 187             )
 188
 189         try:
 190             with open(self._graph_layout_file, "r") as file_read:
 191                 self._graph_layout = load(file_read, Loader=FullLoader)
 192         except IOError as err:
 193             raise RuntimeError(
 194                 f"Not possible to open the file {self._graph_layout_file}\n"
 195                 f"{err}"
 196             )
 197         except YAMLError as err:
 198             raise RuntimeError(
 199                 f"An error occurred while parsing the specification file "
 200                 f"{self._graph_layout_file}\n{err}"
 201             )
 202
 203         try:
 204             with open(self._tooltip_file, "r") as file_read:
 205                 self._tooltips = load(file_read, Loader=FullLoader)
 206         except IOError as err:
 207             logging.warning(
 208                 f"Not possible to open the file {self._tooltip_file}\n{err}"
 209             )
 210         except YAMLError as err:
 211             logging.warning(
 212                 f"An error occurred while parsing the specification file "
 213                 f"{self._tooltip_file}\n{err}"
 214             )
 215
 216         # Callbacks:
 217         if self._app is not None and hasattr(self, "callbacks"):
 218             self.callbacks(self._app)
 219
 220     @property
 221     def html_layout(self):
 222         return self._html_layout
 223
 224     def add_content(self):
 225         """Top level method which generated the web page.
 226
 227         It generates:
 228         - Store for user input data,
 229         - Navigation bar,
 230         - Main area with control panel and ploting area.
 231
 232         If no HTML layout is provided, an error message is displayed instead.
 233
 234         :returns: The HTML div with the whole page.
 235         :rtype: html.Div
 236         """
 237
 238         if self.html_layout and self._tbs:
 239             return html.Div(
 240                 id="div-main",
 241                 className="small",
 242                 children=[
 243                     dbc.Row(
 244                         id="row-navbar",
 245                         class_name="g-0",
 246                         children=[navbar_report((False, True, False, False)), ]
 247                     ),
 248                     dbc.Row(
 249                         id="row-main",
 250                         class_name="g-0",
 251                         children=[
 252                             dcc.Store(id="store-control-panel"),
 253                             dcc.Store(id="store-selected"),
 254                             dcc.Store(id="store-table-data"),
 255                             dcc.Store(id="store-filtered-table-data"),
 256                             dcc.Location(id="url", refresh=False),
 257                             self._add_ctrl_col(),
 258                             self._add_plotting_col()
 259                         ]
 260                     ),
 261                     dbc.Spinner(
 262                         dbc.Offcanvas(
 263                             class_name="w-75",
 264                             id="offcanvas-details",
 265                             title="Test Details",
 266                             placement="end",
 267                             is_open=False,
 268                             children=[]
 269                         ),
 270                         delay_show=C.SPINNER_DELAY
 271                     ),
 272                     dbc.Spinner(
 273                         dbc.Offcanvas(
 274                             class_name="w-50",
 275                             id="offcanvas-metadata",
 276                             title="Detailed Information",
 277                             placement="end",
 278                             is_open=False,
 279                             children=[
 280                                 dbc.Row(id="metadata-tput-lat"),
 281                                 dbc.Row(id="metadata-hdrh-graph")
 282                             ]
 283                         ),
 284                         delay_show=C.SPINNER_DELAY
 285                     ),
 286                     dbc.Offcanvas(
 287                         class_name="w-75",
 288                         id="offcanvas-documentation",
 289                         title="Documentation",
 290                         placement="end",
 291                         is_open=False,
 292                         children=html.Iframe(
 293                             src=C.URL_DOC_REL_NOTES,
 294                             width="100%",
 295                             height="100%"
 296                         )
 297                     )
 298                 ]
 299             )
 300         else:
 301             return html.Div(
 302                 id="div-main-error",
 303                 children=[
 304                     dbc.Alert(
 305                         [
 306                             "An Error Occured"
 307                         ],
 308                         color="danger"
 309                     )
 310                 ]
 311             )
 312
 313     def _add_ctrl_col(self) -> dbc.Col:
 314         """Add column with controls. It is placed on the left side.
 315
 316         :returns: Column with the control panel.
 317         :rtype: dbc.Col
 318         """
 319         return dbc.Col([
 320             html.Div(
 321                 children=self._add_ctrl_panel(),
 322                 className="sticky-top"
 323             )
 324         ])
 325
 326     def _add_plotting_col(self) -> dbc.Col:
 327         """Add column with plots. It is placed on the right side.
 328
 329         :returns: Column with plots.
 330         :rtype: dbc.Col
 331         """
 332         return dbc.Col(
 333             id="col-plotting-area",
 334             children=[
 335                 dbc.Spinner(
 336                     children=[
 337                         dbc.Row(
 338                             id="plotting-area",
 339                             class_name="g-0 p-0",
 340                             children=[
 341                                 C.PLACEHOLDER
 342                             ]
 343                         )
 344                     ]
 345                 )
 346             ],
 347             width=9
 348         )
 349
 350     def _add_ctrl_panel(self) -> list:
 351         """Add control panel.
 352
 353         :returns: Control panel.
 354         :rtype: list
 355         """
 356
 357         reference = [
 358             dbc.Row(
 359                 class_name="g-0 p-1",
 360                 children=[
 361                     dbc.InputGroup(
 362                         [
 363                             dbc.InputGroupText(
 364                                 show_tooltip(self._tooltips, "help-dut", "DUT")
 365                             ),
 366                             dbc.Select(
 367                                 id={"type": "ctrl-dd", "index": "dut"},
 368                                 placeholder="Select a Device under Test...",
 369                                 options=sorted(
 370                                     [
 371                                         {"label": k, "value": k} \
 372                                             for k in self._tbs.keys()
 373                                     ],
 374                                     key=lambda d: d["label"]
 375                                 )
 376                             )
 377                         ],
 378                         size="sm"
 379                     )
 380                 ]
 381             ),
 382             dbc.Row(
 383                 class_name="g-0 p-1",
 384                 children=[
 385                     dbc.InputGroup(
 386                         [
 387                             dbc.InputGroupText(show_tooltip(
 388                                 self._tooltips,
 389                                 "help-csit-dut",
 390                                 "CSIT and DUT Version"
 391                             )),
 392                             dbc.Select(
 393                                 id={"type": "ctrl-dd", "index": "dutver"},
 394                                 placeholder="Select a CSIT and DUT Version...")
 395                         ],
 396                         size="sm"
 397                     )
 398                 ]
 399             ),
 400             dbc.Row(
 401                 class_name="g-0 p-1",
 402                 children=[
 403                     dbc.InputGroup(
 404                         [
 405                             dbc.InputGroupText(show_tooltip(
 406                                 self._tooltips,
 407                                 "help-infra",
 408                                 "Infra"
 409                             )),
 410                             dbc.Select(
 411                                 id={"type": "ctrl-dd", "index": "infra"},
 412                                 placeholder=\
 413                                     "Select a Physical Test Bed Topology..."
 414                             )
 415                         ],
 416                         size="sm"
 417                     )
 418                 ]
 419             ),
 420             dbc.Row(
 421                 class_name="g-0 p-1",
 422                 children=[
 423                     dbc.InputGroup(
 424                         [
 425                             dbc.InputGroupText(show_tooltip(
 426                                 self._tooltips,
 427                                 "help-framesize",
 428                                 "Frame Size"
 429                             )),
 430                             dbc.Checklist(
 431                                 id={"type": "ctrl-cl", "index": "frmsize"},
 432                                 inline=True,
 433                                 class_name="ms-2"
 434                             )
 435                         ],
 436                         style={"align-items": "center"},
 437                         size="sm"
 438                     )
 439                 ]
 440             ),
 441             dbc.Row(
 442                 class_name="g-0 p-1",
 443                 children=[
 444                     dbc.InputGroup(
 445                         [
 446                             dbc.InputGroupText(show_tooltip(
 447                                 self._tooltips,
 448                                 "help-cores",
 449                                 "Number of Cores"
 450                             )),
 451                             dbc.Checklist(
 452                                 id={"type": "ctrl-cl", "index": "core"},
 453                                 inline=True,
 454                                 class_name="ms-2"
 455                             )
 456                         ],
 457                         style={"align-items": "center"},
 458                         size="sm"
 459                     )
 460                 ]
 461             ),
 462             dbc.Row(
 463                 class_name="g-0 p-1",
 464                 children=[
 465                     dbc.InputGroup(
 466                         [
 467                             dbc.InputGroupText(show_tooltip(
 468                                 self._tooltips,
 469                                 "help-measurement",
 470                                 "Measurement"
 471                             )),
 472                             dbc.Checklist(
 473                                 id={"type": "ctrl-cl", "index": "ttype"},
 474                                 inline=True,
 475                                 class_name="ms-2"
 476                             )
 477                         ],
 478                         style={"align-items": "center"},
 479                         size="sm"
 480                     )
 481                 ]
 482             )
 483         ]
 484
 485         compare = [
 486             dbc.Row(
 487                 class_name="g-0 p-1",
 488                 children=[
 489                     dbc.InputGroup(
 490                         [
 491                             dbc.InputGroupText(show_tooltip(
 492                                 self._tooltips,
 493                                 "help-cmp-parameter",
 494                                 "Parameter"
 495                             )),
 496                             dbc.Select(
 497                                 id={"type": "ctrl-dd", "index": "cmpprm"},
 498                                 placeholder="Select a Parameter..."
 499                             )
 500                         ],
 501                         size="sm"
 502                     )
 503                 ]
 504             ),
 505             dbc.Row(
 506                 class_name="g-0 p-1",
 507                 children=[
 508                     dbc.InputGroup(
 509                         [
 510                             dbc.InputGroupText(show_tooltip(
 511                                 self._tooltips,
 512                                 "help-cmp-value",
 513                                 "Value"
 514                             )),
 515                             dbc.Select(
 516                                 id={"type": "ctrl-dd", "index": "cmpval"},
 517                                 placeholder="Select a Value..."
 518                             )
 519                         ],
 520                         size="sm"
 521                     )
 522                 ]
 523             )
 524         ]
 525
 526         processing = [
 527             dbc.Row(
 528                 class_name="g-0 p-1",
 529                 children=[
 530                     dbc.InputGroup(
 531                         children = [
 532                             dbc.Checklist(
 533                                 id="normalize",
 534                                 options=[{
 535                                     "value": "normalize",
 536                                     "label": "Normalize to 2GHz CPU frequency"
 537                                 }],
 538                                 value=[],
 539                                 inline=True,
 540                                 class_name="ms-2"
 541                             ),
 542                             dbc.Checklist(
 543                                 id="outliers",
 544                                 options=[{
 545                                     "value": "outliers",
 546                                     "label": "Remove Extreme Outliers"
 547                                 }],
 548                                 value=[],
 549                                 inline=True,
 550                                 class_name="ms-2"
 551                             )
 552                         ],
 553                         style={"align-items": "center"},
 554                         size="sm"
 555                     )
 556                 ]
 557             )
 558         ]
 559
 560         return [
 561             dbc.Row(
 562                 dbc.Card(
 563                     [
 564                         dbc.CardHeader(
 565                             html.H5("Reference Value")
 566                         ),
 567                         dbc.CardBody(
 568                             children=reference,
 569                             class_name="g-0 p-0"
 570                         )
 571                     ],
 572                     color="secondary",
 573                     outline=True
 574                 ),
 575                 class_name="g-0 p-1"
 576             ),
 577             dbc.Row(
 578                 dbc.Card(
 579                     [
 580                         dbc.CardHeader(
 581                             html.H5("Compared Value")
 582                         ),
 583                         dbc.CardBody(
 584                             children=compare,
 585                             class_name="g-0 p-0"
 586                         )
 587                     ],
 588                     color="secondary",
 589                     outline=True
 590                 ),
 591                 class_name="g-0 p-1"
 592             ),
 593             dbc.Row(
 594                 dbc.Card(
 595                     [
 596                         dbc.CardHeader(
 597                             html.H5("Data Manipulations")
 598                         ),
 599                         dbc.CardBody(
 600                             children=processing,
 601                             class_name="g-0 p-0"
 602                         )
 603                     ],
 604                     color="secondary",
 605                     outline=True
 606                 ),
 607                 class_name="g-0 p-1"
 608             )
 609         ]
 610
 611     @staticmethod
 612     def _get_plotting_area(
 613             title: str,
 614             table: pd.DataFrame,
 615             url: str
 616         ) -> list:
 617         """Generate the plotting area with all its content.
 618
 619         :param title: The title of the comparison table.
 620         :param table: Comparison table to be displayed.
 621         :param url: URL to be displayed in the modal window.
 622         :type title: str
 623         :type table: pandas.DataFrame
 624         :type url: str
 625         :returns: List of rows with elements to be displayed in the plotting
 626             area.
 627         :rtype: list
 628         """
 629
 630         if table.empty:
 631             return dbc.Row(
 632                 dbc.Col(
 633                     children=dbc.Alert(
 634                         "No data for comparison.",
 635                         color="danger"
 636                     ),
 637                     class_name="g-0 p-1",
 638                 ),
 639                 class_name="g-0 p-0"
 640             )
 641
 642         cols = list()
 643         for idx, col in enumerate(table.columns):
 644             if idx == 0:
 645                 cols.append({
 646                     "name": ["", col],
 647                     "id": col,
 648                     "deletable": False,
 649                     "selectable": False,
 650                     "type": "text"
 651                 })
 652             else:
 653                 l_col = col.rsplit(" ", 2)
 654                 cols.append({
 655                     "name": [l_col[0], " ".join(l_col[-2:])],
 656                     "id": col,
 657                     "deletable": False,
 658                     "selectable": False,
 659                     "type": "numeric",
 660                     "format": Format(precision=2, scheme=Scheme.fixed)
 661                 })
 662
 663         return [
 664             dbc.Row(
 665                 children=html.H5(title),
 666                 class_name="g-0 p-1"
 667             ),
 668             dbc.Row(
 669                 children=[
 670                     dbc.Col(
 671                         children=dash_table.DataTable(
 672                             id={"type": "table", "index": "comparison"},
 673                             columns=cols,
 674                             data=table.to_dict("records"),
 675                             merge_duplicate_headers=True,
 676                             editable=False,
 677                             filter_action="custom",
 678                             filter_query="",
 679                             sort_action="custom",
 680                             sort_mode="multi",
 681                             selected_columns=[],
 682                             selected_rows=[],
 683                             page_action="none",
 684                             style_cell={"textAlign": "right"},
 685                             style_cell_conditional=[{
 686                                 "if": {"column_id": "Test Name"},
 687                                 "textAlign": "left"
 688                             }]
 689                         ),
 690                         class_name="g-0 p-1"
 691                     )
 692                 ],
 693                 class_name="g-0 p-0"
 694             ),
 695             dbc.Row(
 696                 [
 697                     dbc.Col([html.Div(
 698                         [
 699                             dbc.Button(
 700                                 id="plot-btn-url",
 701                                 children="Show URL",
 702                                 class_name="me-1",
 703                                 color="info",
 704                                 style={
 705                                     "text-transform": "none",
 706                                     "padding": "0rem 1rem"
 707                                 }
 708                             ),
 709                             dbc.Modal(
 710                                 [
 711                                     dbc.ModalHeader(dbc.ModalTitle("URL")),
 712                                     dbc.ModalBody(url)
 713                                 ],
 714                                 id="plot-mod-url",
 715                                 size="xl",
 716                                 is_open=False,
 717                                 scrollable=True
 718                             ),
 719                             dbc.Button(
 720                                 id="plot-btn-download",
 721                                 children="Download Table",
 722                                 class_name="me-1",
 723                                 color="info",
 724                                 style={
 725                                     "text-transform": "none",
 726                                     "padding": "0rem 1rem"
 727                                 }
 728                             ),
 729                             dcc.Download(id="download-iterative-data"),
 730                             dbc.Button(
 731                                 id="plot-btn-download-raw",
 732                                 children="Download Raw Data",
 733                                 class_name="me-1",
 734                                 color="info",
 735                                 style={
 736                                     "text-transform": "none",
 737                                     "padding": "0rem 1rem"
 738                                 }
 739                             ),
 740                             dcc.Download(id="download-raw-data")
 741                         ],
 742                         className=\
 743                             "d-grid gap-0 d-md-flex justify-content-md-end"
 744                     )])
 745                 ],
 746                 class_name="g-0 p-0"
 747             ),
 748             dbc.Row(
 749                 children=C.PLACEHOLDER,
 750                 class_name="g-0 p-1"
 751             )
 752         ]
 753
 754     def callbacks(self, app):
 755         """Callbacks for the whole application.
 756
 757         :param app: The application.
 758         :type app: Flask
 759         """
 760
 761         @app.callback(
 762             [
 763                 Output("store-control-panel", "data"),
 764                 Output("store-selected", "data"),
 765                 Output("store-table-data", "data"),
 766                 Output("store-filtered-table-data", "data"),
 767                 Output("plotting-area", "children"),
 768                 Output({"type": "table", "index": ALL}, "data"),
 769                 Output({"type": "ctrl-dd", "index": "dut"}, "value"),
 770                 Output({"type": "ctrl-dd", "index": "dutver"}, "options"),
 771                 Output({"type": "ctrl-dd", "index": "dutver"}, "disabled"),
 772                 Output({"type": "ctrl-dd", "index": "dutver"}, "value"),
 773                 Output({"type": "ctrl-dd", "index": "infra"}, "options"),
 774                 Output({"type": "ctrl-dd", "index": "infra"}, "disabled"),
 775                 Output({"type": "ctrl-dd", "index": "infra"}, "value"),
 776                 Output({"type": "ctrl-cl", "index": "core"}, "options"),
 777                 Output({"type": "ctrl-cl", "index": "core"}, "value"),
 778                 Output({"type": "ctrl-cl", "index": "frmsize"}, "options"),
 779                 Output({"type": "ctrl-cl", "index": "frmsize"}, "value"),
 780                 Output({"type": "ctrl-cl", "index": "ttype"}, "options"),
 781                 Output({"type": "ctrl-cl", "index": "ttype"}, "value"),
 782                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "options"),
 783                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "disabled"),
 784                 Output({"type": "ctrl-dd", "index": "cmpprm"}, "value"),
 785                 Output({"type": "ctrl-dd", "index": "cmpval"}, "options"),
 786                 Output({"type": "ctrl-dd", "index": "cmpval"}, "disabled"),
 787                 Output({"type": "ctrl-dd", "index": "cmpval"}, "value"),
 788                 Output("normalize", "value"),
 789                 Output("outliers", "value")
 790             ],
 791             [
 792                 State("store-control-panel", "data"),
 793                 State("store-selected", "data"),
 794                 State("store-table-data", "data"),
 795                 State("store-filtered-table-data", "data"),
 796                 State({"type": "table", "index": ALL}, "data")
 797             ],
 798             [
 799                 Input("url", "href"),
 800                 Input("normalize", "value"),
 801                 Input("outliers", "value"),
 802                 Input({"type": "table", "index": ALL}, "filter_query"),
 803                 Input({"type": "table", "index": ALL}, "sort_by"),
 804                 Input({"type": "ctrl-dd", "index": ALL}, "value"),
 805                 Input({"type": "ctrl-cl", "index": ALL}, "value"),
 806                 Input({"type": "ctrl-btn", "index": ALL}, "n_clicks")
 807             ]
 808         )
 809         def _update_application(
 810                 control_panel: dict,
 811                 selected: dict,
 812                 store_table_data: list,
 813                 filtered_data: list,
 814                 table_data: list,
 815                 href: str,
 816                 normalize: list,
 817                 outliers: bool,
 818                 *_
 819             ) -> tuple:
 820             """Update the application when the event is detected.
 821             """
 822
 823             ctrl_panel = ControlPanel(CP_PARAMS, control_panel)
 824
 825             if selected is None:
 826                 selected = {
 827                     "reference": {
 828                         "set": False,
 829                     },
 830                     "compare": {
 831                         "set": False,
 832                     }
 833                 }
 834
 835             # Parse the url:
 836             parsed_url = url_decode(href)
 837             if parsed_url:
 838                 url_params = parsed_url["params"]
 839             else:
 840                 url_params = None
 841
 842             on_draw = False
 843             plotting_area = no_update
 844
 845             trigger = Trigger(callback_context.triggered)
 846             if trigger.type == "url" and url_params:
 847                 process_url = False
 848                 try:
 849                     selected = literal_eval(url_params["selected"][0])
 850                     r_sel = selected["reference"]["selection"]
 851                     c_sel = selected["compare"]
 852                     normalize = literal_eval(url_params["norm"][0])
 853                     try:  # Necessary for backward compatibility
 854                         outliers = literal_eval(url_params["outliers"][0])
 855                     except (KeyError, IndexError, AttributeError):
 856                         outliers = list()
 857                     process_url = bool(
 858                         (selected["reference"]["set"] == True) and
 859                         (c_sel["set"] == True)
 860                     )
 861                 except (KeyError, IndexError, AttributeError):
 862                     pass
 863                 if process_url:
 864                     ctrl_panel.set({
 865                         "dut-val": r_sel["dut"],
 866                         "dutver-opt": generate_options(
 867                             self._tbs[r_sel["dut"]].keys()
 868                         ),
 869                         "dutver-dis": False,
 870                         "dutver-val": r_sel["dutver"],
 871                         "infra-opt": generate_options(
 872                             self._tbs[r_sel["dut"]][r_sel["dutver"]].keys()
 873                         ),
 874                         "infra-dis": False,
 875                         "infra-val": r_sel["infra"],
 876                         "core-opt": generate_options(
 877                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 878                                 [r_sel["infra"]]["core"]
 879                         ),
 880                         "core-val": r_sel["core"],
 881                         "frmsize-opt": generate_options(
 882                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 883                                 [r_sel["infra"]]["fsize"]
 884                         ),
 885                         "frmsize-val": r_sel["frmsize"],
 886                         "ttype-opt": generate_options(
 887                             self._tbs[r_sel["dut"]][r_sel["dutver"]]\
 888                                 [r_sel["infra"]]["ttype"]
 889                         ),
 890                         "ttype-val": r_sel["ttype"],
 891                         "normalize-val": normalize,
 892                         "outliers-val": outliers
 893                     })
 894                     opts = list()
 895                     for itm, label in CMP_PARAMS.items():
 896                         if len(ctrl_panel.get(f"{itm}-opt")) > 1:
 897                             opts.append({"label": label, "value": itm})
 898                     ctrl_panel.set({
 899                         "cmp-par-opt": opts,
 900                         "cmp-par-dis": False,
 901                         "cmp-par-val": c_sel["parameter"]
 902                     })
 903                     opts = list()
 904                     for itm in ctrl_panel.get(f"{c_sel['parameter']}-opt"):
 905                         set_val = ctrl_panel.get(f"{c_sel['parameter']}-val")
 906                         if isinstance(set_val, list):
 907                             if itm["value"] not in set_val:
 908                                 opts.append(itm)
 909                         else:
 910                             if itm["value"] != set_val:
 911                                 opts.append(itm)
 912                     ctrl_panel.set({
 913                         "cmp-val-opt": opts,
 914                         "cmp-val-dis": False,
 915                         "cmp-val-val": c_sel["value"]
 916                     })
 917                     on_draw = True
 918             elif trigger.type == "normalize":
 919                 ctrl_panel.set({"normalize-val": normalize})
 920                 on_draw = True
 921             elif trigger.type == "outliers":
 922                 ctrl_panel.set({"outliers-val": outliers})
 923                 on_draw = True
 924             elif trigger.type == "ctrl-dd":
 925                 if trigger.idx == "dut":
 926                     try:
 927                         opts = generate_options(self._tbs[trigger.value].keys())
 928                         disabled = False
 929                     except KeyError:
 930                         opts = list()
 931                         disabled = True
 932                     ctrl_panel.set({
 933                         "dut-val": trigger.value,
 934                         "dutver-opt": opts,
 935                         "dutver-dis": disabled,
 936                         "dutver-val": str(),
 937                         "infra-opt": list(),
 938                         "infra-dis": True,
 939                         "infra-val": str(),
 940                         "core-opt": list(),
 941                         "core-val": list(),
 942                         "frmsize-opt": list(),
 943                         "frmsize-val": list(),
 944                         "ttype-opt": list(),
 945                         "ttype-val": list(),
 946                         "cmp-par-opt": list(),
 947                         "cmp-par-dis": True,
 948                         "cmp-par-val": str(),
 949                         "cmp-val-opt": list(),
 950                         "cmp-val-dis": True,
 951                         "cmp-val-val": str()
 952                     })
 953                 elif trigger.idx == "dutver":
 954                     try:
 955                         dut = ctrl_panel.get("dut-val")
 956                         dver = self._tbs[dut][trigger.value]
 957                         opts = generate_options(dver.keys())
 958                         disabled = False
 959                     except KeyError:
 960                         opts = list()
 961                         disabled = True
 962                     ctrl_panel.set({
 963                         "dutver-val": trigger.value,
 964                         "infra-opt": opts,
 965                         "infra-dis": disabled,
 966                         "infra-val": str(),
 967                         "core-opt": list(),
 968                         "core-val": list(),
 969                         "frmsize-opt": list(),
 970                         "frmsize-val": list(),
 971                         "ttype-opt": list(),
 972                         "ttype-val": list(),
 973                         "cmp-par-opt": list(),
 974                         "cmp-par-dis": True,
 975                         "cmp-par-val": str(),
 976                         "cmp-val-opt": list(),
 977                         "cmp-val-dis": True,
 978                         "cmp-val-val": str()
 979                     })
 980                 elif trigger.idx == "infra":
 981                     dut = ctrl_panel.get("dut-val")
 982                     dver = ctrl_panel.get("dutver-val")
 983                     if all((dut, dver, trigger.value, )):
 984                         driver = self._tbs[dut][dver][trigger.value]
 985                         ctrl_panel.set({
 986                             "infra-val": trigger.value,
 987                             "core-opt": generate_options(driver["core"]),
 988                             "core-val": list(),
 989                             "frmsize-opt": generate_options(driver["fsize"]),
 990                             "frmsize-val": list(),
 991                             "ttype-opt": generate_options(driver["ttype"]),
 992                             "ttype-val": list(),
 993                             "cmp-par-opt": list(),
 994                             "cmp-par-dis": True,
 995                             "cmp-par-val": str(),
 996                             "cmp-val-opt": list(),
 997                             "cmp-val-dis": True,
 998                             "cmp-val-val": str()
 999                         })
1000                 elif trigger.idx == "cmpprm":
1001                     value = trigger.value
1002                     opts = list()
1003                     for itm in ctrl_panel.get(f"{value}-opt"):
1004                         set_val = ctrl_panel.get(f"{value}-val")
1005                         if isinstance(set_val, list):
1006                             if itm["value"] == "Latency":
1007                                 continue
1008                             if itm["value"] not in set_val:
1009                                 opts.append(itm)
1010                         else:
1011                             if itm["value"] != set_val:
1012                                 opts.append(itm)
1013                     ctrl_panel.set({
1014                         "cmp-par-val": value,
1015                         "cmp-val-opt": opts,
1016                         "cmp-val-dis": False,
1017                         "cmp-val-val": str()
1018                     })
1019                 elif trigger.idx == "cmpval":
1020                     ctrl_panel.set({"cmp-val-val": trigger.value})
1021                     selected["reference"] = {
1022                         "set": True,
1023                         "selection": {
1024                             "dut": ctrl_panel.get("dut-val"),
1025                             "dutver": ctrl_panel.get("dutver-val"),
1026                             "infra": ctrl_panel.get("infra-val"),
1027                             "core": ctrl_panel.get("core-val"),
1028                             "frmsize": ctrl_panel.get("frmsize-val"),
1029                             "ttype": ctrl_panel.get("ttype-val")
1030                         }
1031                     }
1032                     selected["compare"] = {
1033                         "set": True,
1034                         "parameter": ctrl_panel.get("cmp-par-val"),
1035                         "value": trigger.value
1036                     }
1037                     on_draw = True
1038             elif trigger.type == "ctrl-cl":
1039                 ctrl_panel.set({f"{trigger.idx}-val": trigger.value})
1040                 if all((ctrl_panel.get("core-val"),
1041                         ctrl_panel.get("frmsize-val"),
1042                         ctrl_panel.get("ttype-val"), )):
1043                     if "Latency" in ctrl_panel.get("ttype-val"):
1044                         ctrl_panel.set({"ttype-val": ["Latency", ]})
1045                     opts = list()
1046                     for itm, label in CMP_PARAMS.items():
1047                         if "Latency" in ctrl_panel.get("ttype-val") and \
1048                                 itm == "ttype":
1049                             continue
1050                         if len(ctrl_panel.get(f"{itm}-opt")) > 1:
1051                             if isinstance(ctrl_panel.get(f"{itm}-val"), list):
1052                                 if len(ctrl_panel.get(f"{itm}-opt")) == \
1053                                         len(ctrl_panel.get(f"{itm}-val")):
1054                                     continue
1055                             opts.append({"label": label, "value": itm})
1056                     ctrl_panel.set({
1057                         "cmp-par-opt": opts,
1058                         "cmp-par-dis": False,
1059                         "cmp-par-val": str(),
1060                         "cmp-val-opt": list(),
1061                         "cmp-val-dis": True,
1062                         "cmp-val-val": str()
1063                     })
1064                 else:
1065                     ctrl_panel.set({
1066                         "cmp-par-opt": list(),
1067                         "cmp-par-dis": True,
1068                         "cmp-par-val": str(),
1069                         "cmp-val-opt": list(),
1070                         "cmp-val-dis": True,
1071                         "cmp-val-val": str()
1072                     })
1073             elif trigger.type == "table" and trigger.idx == "comparison":
1074                 if trigger.parameter == "filter_query":
1075                     filtered_data = filter_table_data(
1076                         store_table_data,
1077                         trigger.value
1078                     )
1079                 elif trigger.parameter == "sort_by":
1080                     filtered_data = sort_table_data(
1081                         store_table_data,
1082                         trigger.value
1083                     )
1084                 table_data = [filtered_data, ]
1085
1086             if all((on_draw, selected["reference"]["set"],
1087                     selected["compare"]["set"], )):
1088                 title, table = comparison_table(
1089                     data=self._data,
1090                     selected=selected,
1091                     normalize=normalize,
1092                     format="html",
1093                     remove_outliers=outliers
1094                 )
1095                 plotting_area = self._get_plotting_area(
1096                     title=title,
1097                     table=table,
1098                     url=gen_new_url(
1099                         parsed_url,
1100                         params={
1101                             "selected": selected,
1102                             "norm": normalize,
1103                             "outliers": outliers
1104                         }
1105                     )
1106                 )
1107                 store_table_data = table.to_dict("records")
1108                 filtered_data = store_table_data
1109                 if table_data:
1110                     table_data = [store_table_data, ]
1111
1112             ret_val = [
1113                 ctrl_panel.panel,
1114                 selected,
1115                 store_table_data,
1116                 filtered_data,
1117                 plotting_area,
1118                 table_data
1119             ]
1120             ret_val.extend(ctrl_panel.values)
1121             return ret_val
1122
1123         @app.callback(
1124             Output("plot-mod-url", "is_open"),
1125             Input("plot-btn-url", "n_clicks"),
1126             State("plot-mod-url", "is_open")
1127         )
1128         def toggle_plot_mod_url(n, is_open):
1129             """Toggle the modal window with url.
1130             """
1131             if n:
1132                 return not is_open
1133             return is_open
1134
1135         @app.callback(
1136             Output("download-iterative-data", "data"),
1137             State("store-table-data", "data"),
1138             State("store-filtered-table-data", "data"),
1139             Input("plot-btn-download", "n_clicks"),
1140             prevent_initial_call=True
1141         )
1142         def _download_comparison_data(
1143                 table_data: list,
1144                 filtered_table_data: list,
1145                 _: int
1146             ) -> dict:
1147             """Download the data.
1148
1149             :param table_data: Original unfiltered table data.
1150             :param filtered_table_data: Filtered table data.
1151             :type table_data: list
1152             :type filtered_table_data: list
1153             :returns: dict of data frame content (base64 encoded) and meta data
1154                 used by the Download component.
1155             :rtype: dict
1156             """
1157
1158             if not table_data:
1159                 raise PreventUpdate
1160
1161             if filtered_table_data:
1162                 table = pd.DataFrame.from_records(filtered_table_data)
1163             else:
1164                 table = pd.DataFrame.from_records(table_data)
1165
1166             return dcc.send_data_frame(table.to_csv, C.COMP_DOWNLOAD_FILE_NAME)
1167
1168         @app.callback(
1169             Output("download-raw-data", "data"),
1170             State("store-selected", "data"),
1171             Input("plot-btn-download-raw", "n_clicks"),
1172             prevent_initial_call=True
1173         )
1174         def _download_raw_comparison_data(selected: dict, _: int) -> dict:
1175             """Download the data.
1176
1177             :param selected: Selected tests.
1178             :type selected: dict
1179             :returns: dict of data frame content (base64 encoded) and meta data
1180                 used by the Download component.
1181             :rtype: dict
1182             """
1183
1184             if not selected:
1185                 raise PreventUpdate
1186
1187             _, table = comparison_table(
1188                     data=self._data,
1189                     selected=selected,
1190                     normalize=False,
1191                     remove_outliers=False,
1192                     raw_data=True
1193                 )
1194
1195             return dcc.send_data_frame(
1196                 table.dropna(how="all", axis=1).to_csv,
1197                 f"raw_{C.COMP_DOWNLOAD_FILE_NAME}"
1198             )
1199
1200         @app.callback(
1201             Output("offcanvas-documentation", "is_open"),
1202             Input("btn-documentation", "n_clicks"),
1203             State("offcanvas-documentation", "is_open")
1204         )
1205         def toggle_offcanvas_documentation(n_clicks, is_open):
1206             if n_clicks:
1207                 return not is_open
1208             return is_open
1209
1210         @app.callback(
1211             Output("offcanvas-details", "is_open"),
1212             Output("offcanvas-details", "children"),
1213             State("store-selected", "data"),
1214             State("store-filtered-table-data", "data"),
1215             State("normalize", "value"),
1216             State("outliers", "value"),
1217             Input({"type": "table", "index": ALL}, "active_cell"),
1218             prevent_initial_call=True
1219         )
1220         def show_test_data(cp_sel, table, normalize, outliers, *_):
1221             """Show offcanvas with graphs and tables based on selected test(s).
1222             """
1223
1224             trigger = Trigger(callback_context.triggered)
1225             if not all((trigger.value, cp_sel["reference"]["set"], \
1226                         cp_sel["compare"]["set"])):
1227                 raise PreventUpdate
1228
1229             try:
1230                 test_name = pd.DataFrame.from_records(table).\
1231                     iloc[[trigger.value["row"]]]["Test Name"].iloc[0]
1232                 dut = cp_sel["reference"]["selection"]["dut"]
1233                 rls, dutver = cp_sel["reference"]["selection"]["dutver"].\
1234                     split("-", 1)
1235                 phy = cp_sel["reference"]["selection"]["infra"]
1236                 framesize, core, test_id = test_name.split("-", 2)
1237                 test, ttype = test_id.rsplit("-", 1)
1238                 ttype = "pdr" if ttype == "latency" else ttype
1239                 l_phy = phy.split("-")
1240                 tb = "-".join(l_phy[:2])
1241                 nic = l_phy[2]
1242                 stype = "ndrpdr" if ttype in ("ndr", "pdr") else ttype
1243             except(KeyError, IndexError, AttributeError, ValueError):
1244                 raise PreventUpdate
1245
1246             df = pd.DataFrame(self._data.loc[(
1247                     (self._data["dut_type"] == dut) &
1248                     (self._data["dut_version"] == dutver) &
1249                     (self._data["release"] == rls)
1250                 )])
1251             df = df[df.job.str.endswith(tb)]
1252             df = df[df.test_id.str.contains(
1253                 f"{nic}.*{test}-{stype}", regex=True
1254             )]
1255             if df.empty:
1256                 raise PreventUpdate
1257
1258             l_test_id = df["test_id"].iloc[0].split(".")
1259             area = ".".join(l_test_id[3:-2])
1260
1261             r_sel = {
1262                 "id": f"{test}-{ttype}",
1263                 "rls": rls,
1264                 "dut": dut,
1265                 "dutver": dutver,
1266                 "phy": phy,
1267                 "area": area,
1268                 "test": test,
1269                 "framesize": framesize,
1270                 "core": core,
1271                 "testtype": ttype
1272             }
1273
1274             c_sel = deepcopy(r_sel)
1275             param = cp_sel["compare"]["parameter"]
1276             val = cp_sel["compare"]["value"].lower()
1277             if param == "dutver":
1278                 c_sel["rls"], c_sel["dutver"] = val.split("-", 1)
1279             elif param == "ttype":
1280                 c_sel["id"] = f"{test}-{val}"
1281                 c_sel["testtype"] = val
1282             elif param == "infra":
1283                 c_sel["phy"] = val
1284             else:
1285                 c_sel[param] = val
1286
1287             r_sel["id"] = "-".join(
1288                 (r_sel["phy"], r_sel["framesize"], r_sel["core"], r_sel["id"])
1289             )
1290             c_sel["id"] = "-".join(
1291                 (c_sel["phy"], c_sel["framesize"], c_sel["core"], c_sel["id"])
1292             )
1293             selected = [r_sel, c_sel]
1294
1295             indexes = ("tput", "bandwidth", "lat")
1296             graphs = graph_iterative(
1297                 self._data,
1298                 selected,
1299                 self._graph_layout,
1300                 bool(normalize),
1301                 bool(outliers)
1302             )
1303             cols = list()
1304             for graph, idx in zip(graphs, indexes):
1305                 if graph:
1306                     cols.append(dbc.Col(dcc.Graph(
1307                         figure=graph,
1308                         id={"type": "graph-iter", "index": idx},
1309                     )))
1310             if not cols:
1311                 cols="No data."
1312             ret_val = [
1313                 dbc.Row(
1314                     class_name="g-0 p-0",
1315                     children=dbc.Alert(test, color="info"),
1316                 ),
1317                 dbc.Row(class_name="g-0 p-0", children=cols)
1318             ]
1319
1320             return True, ret_val
1321
1322         @app.callback(
1323             Output("metadata-tput-lat", "children"),
1324             Output("metadata-hdrh-graph", "children"),
1325             Output("offcanvas-metadata", "is_open"),
1326             Input({"type": "graph-iter", "index": ALL}, "clickData"),
1327             prevent_initial_call=True
1328         )
1329         def _show_metadata_from_graph(iter_data: dict) -> tuple:
1330             """Generates the data for the offcanvas displayed when a particular
1331             point in a graph is clicked on.
1332             """
1333
1334             trigger = Trigger(callback_context.triggered)
1335             if not trigger.value:
1336                 raise PreventUpdate
1337
1338             if trigger.type == "graph-iter":
1339                 return show_iterative_graph_data(
1340                     trigger, iter_data, self._graph_layout)
1341             else:
1342                 raise PreventUpdate