resources/tools/dash/app/pal/report/layout.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import logging
  18 import pandas as pd
  19 import dash_bootstrap_components as dbc
  20
  21 from flask import Flask
  22 from dash import dcc
  23 from dash import html
  24 from dash import callback_context, no_update, ALL
  25 from dash import Input, Output, State
  26 from yaml import load, FullLoader, YAMLError
  27 from copy import deepcopy
  28 from ast import literal_eval
  29
  30 from ..utils.constants import Constants as C
  31 from ..utils.utils import show_tooltip, label, sync_checklists, list_tests, \
  32     gen_new_url
  33 from ..utils.url_processing import url_decode
  34 from ..data.data import Data
  35 from .graphs import graph_iterative, table_comparison, get_short_version
  36
  37
  38 class Layout:
  39     """
  40     """
  41
  42     def __init__(self, app: Flask, releases: list, html_layout_file: str,
  43         graph_layout_file: str, data_spec_file: str, tooltip_file: str) -> None:
  44         """
  45         """
  46
  47         # Inputs
  48         self._app = app
  49         self.releases = releases
  50         self._html_layout_file = html_layout_file
  51         self._graph_layout_file = graph_layout_file
  52         self._data_spec_file = data_spec_file
  53         self._tooltip_file = tooltip_file
  54
  55         # Read the data:
  56         self._data = pd.DataFrame()
  57         for rls in releases:
  58             data_mrr = Data(self._data_spec_file, True).\
  59                 read_iterative_mrr(release=rls.replace("csit", "rls"))
  60             data_mrr["release"] = rls
  61             data_ndrpdr = Data(self._data_spec_file, True).\
  62                 read_iterative_ndrpdr(release=rls.replace("csit", "rls"))
  63             data_ndrpdr["release"] = rls
  64             self._data = pd.concat(
  65                 [self._data, data_mrr, data_ndrpdr], ignore_index=True)
  66
  67         # Get structure of tests:
  68         tbs = dict()
  69         cols = ["job", "test_id", "test_type", "dut_version", "release"]
  70         for _, row in self._data[cols].drop_duplicates().iterrows():
  71             rls = row["release"]
  72             ttype = row["test_type"]
  73             lst_job = row["job"].split("-")
  74             dut = lst_job[1]
  75             d_ver = get_short_version(row["dut_version"], dut)
  76             tbed = "-".join(lst_job[-2:])
  77             lst_test_id = row["test_id"].split(".")
  78             if dut == "dpdk":
  79                 area = "dpdk"
  80             else:
  81                 area = "-".join(lst_test_id[3:-2])
  82             suite = lst_test_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
  83                 replace("2n-", "")
  84             test = lst_test_id[-1]
  85             nic = suite.split("-")[0]
  86             for drv in C.DRIVERS:
  87                 if drv in test:
  88                     driver = drv.replace("-", "_")
  89                     test = test.replace(f"{drv}-", "")
  90                     break
  91             else:
  92                 driver = "dpdk"
  93             infra = "-".join((tbed, nic, driver))
  94             lst_test = test.split("-")
  95             framesize = lst_test[0]
  96             core = lst_test[1] if lst_test[1] else "8C"
  97             test = "-".join(lst_test[2: -1])
  98
  99             if tbs.get(rls, None) is None:
 100                 tbs[rls] = dict()
 101             if tbs[rls].get(dut, None) is None:
 102                 tbs[rls][dut] = dict()
 103             if tbs[rls][dut].get(d_ver, None) is None:
 104                 tbs[rls][dut][d_ver] = dict()
 105             if tbs[rls][dut][d_ver].get(infra, None) is None:
 106                 tbs[rls][dut][d_ver][infra] = dict()
 107             if tbs[rls][dut][d_ver][infra].get(area, None) is None:
 108                 tbs[rls][dut][d_ver][infra][area] = dict()
 109             if tbs[rls][dut][d_ver][infra][area].get(test, None) is None:
 110                 tbs[rls][dut][d_ver][infra][area][test] = dict()
 111                 tbs[rls][dut][d_ver][infra][area][test]["core"] = list()
 112                 tbs[rls][dut][d_ver][infra][area][test]["frame-size"] = list()
 113                 tbs[rls][dut][d_ver][infra][area][test]["test-type"] = list()
 114             if core.upper() not in \
 115                     tbs[rls][dut][d_ver][infra][area][test]["core"]:
 116                 tbs[rls][dut][d_ver][infra][area][test]["core"].append(
 117                     core.upper())
 118             if framesize.upper() not in \
 119                         tbs[rls][dut][d_ver][infra][area][test]["frame-size"]:
 120                 tbs[rls][dut][d_ver][infra][area][test]["frame-size"].append(
 121                     framesize.upper())
 122             if ttype == "mrr":
 123                 if "MRR" not in \
 124                         tbs[rls][dut][d_ver][infra][area][test]["test-type"]:
 125                     tbs[rls][dut][d_ver][infra][area][test]["test-type"].append(
 126                         "MRR")
 127             elif ttype == "ndrpdr":
 128                 if "NDR" not in \
 129                         tbs[rls][dut][d_ver][infra][area][test]["test-type"]:
 130                     tbs[rls][dut][d_ver][infra][area][test]["test-type"].extend(
 131                         ("NDR", "PDR", ))
 132         self._spec_tbs = tbs
 133
 134         # Read from files:
 135         self._html_layout = ""
 136         self._graph_layout = None
 137         self._tooltips = dict()
 138
 139         try:
 140             with open(self._html_layout_file, "r") as file_read:
 141                 self._html_layout = file_read.read()
 142         except IOError as err:
 143             raise RuntimeError(
 144                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 145             )
 146
 147         try:
 148             with open(self._graph_layout_file, "r") as file_read:
 149                 self._graph_layout = load(file_read, Loader=FullLoader)
 150         except IOError as err:
 151             raise RuntimeError(
 152                 f"Not possible to open the file {self._graph_layout_file}\n"
 153                 f"{err}"
 154             )
 155         except YAMLError as err:
 156             raise RuntimeError(
 157                 f"An error occurred while parsing the specification file "
 158                 f"{self._graph_layout_file}\n{err}"
 159             )
 160
 161         try:
 162             with open(self._tooltip_file, "r") as file_read:
 163                 self._tooltips = load(file_read, Loader=FullLoader)
 164         except IOError as err:
 165             logging.warning(
 166                 f"Not possible to open the file {self._tooltip_file}\n{err}"
 167             )
 168         except YAMLError as err:
 169             logging.warning(
 170                 f"An error occurred while parsing the specification file "
 171                 f"{self._tooltip_file}\n{err}"
 172             )
 173
 174         # Callbacks:
 175         if self._app is not None and hasattr(self, 'callbacks'):
 176             self.callbacks(self._app)
 177
 178     @property
 179     def html_layout(self):
 180         return self._html_layout
 181
 182     @property
 183     def spec_tbs(self):
 184         return self._spec_tbs
 185
 186     @property
 187     def data(self):
 188         return self._data
 189
 190     @property
 191     def layout(self):
 192         return self._graph_layout
 193
 194     def add_content(self):
 195         """
 196         """
 197         if self.html_layout and self.spec_tbs:
 198             return html.Div(
 199                 id="div-main",
 200                 children=[
 201                     dbc.Row(
 202                         id="row-navbar",
 203                         class_name="g-0",
 204                         children=[
 205                             self._add_navbar(),
 206                         ]
 207                     ),
 208                     dcc.Loading(
 209                         dbc.Offcanvas(
 210                             class_name="w-50",
 211                             id="offcanvas-metadata",
 212                             title="Throughput And Latency",
 213                             placement="end",
 214                             is_open=False,
 215                             children=[
 216                                 dbc.Row(id="metadata-tput-lat"),
 217                                 dbc.Row(id="metadata-hdrh-graph"),
 218                             ]
 219                         )
 220                     ),
 221                     dbc.Row(
 222                         id="row-main",
 223                         class_name="g-0",
 224                         children=[
 225                             dcc.Store(id="selected-tests"),
 226                             dcc.Store(id="control-panel"),
 227                             dcc.Location(id="url", refresh=False),
 228                             self._add_ctrl_col(),
 229                             self._add_plotting_col(),
 230                         ]
 231                     )
 232                 ]
 233             )
 234         else:
 235             return html.Div(
 236                 id="div-main-error",
 237                 children=[
 238                     dbc.Alert(
 239                         [
 240                             "An Error Occured",
 241                         ],
 242                         color="danger",
 243                     ),
 244                 ]
 245             )
 246
 247     def _add_navbar(self):
 248         """Add nav element with navigation panel. It is placed on the top.
 249         """
 250         return dbc.NavbarSimple(
 251             id="navbarsimple-main",
 252             children=[
 253                 dbc.NavItem(
 254                     dbc.NavLink(
 255                         "Iterative Test Runs",
 256                         disabled=True,
 257                         external_link=True,
 258                         href="#"
 259                     )
 260                 )
 261             ],
 262             brand="Dashboard",
 263             brand_href="/",
 264             brand_external_link=True,
 265             class_name="p-2",
 266             fluid=True,
 267         )
 268
 269     def _add_ctrl_col(self) -> dbc.Col:
 270         """Add column with controls. It is placed on the left side.
 271         """
 272         return dbc.Col(
 273             id="col-controls",
 274             children=[
 275                 self._add_ctrl_panel(),
 276             ],
 277         )
 278
 279     def _add_plotting_col(self) -> dbc.Col:
 280         """Add column with plots and tables. It is placed on the right side.
 281         """
 282         return dbc.Col(
 283             id="col-plotting-area",
 284             children=[
 285                 dcc.Loading(
 286                     children=[
 287                         dbc.Row(  # Graphs
 288                             class_name="g-0 p-2",
 289                             children=[
 290                                 dbc.Col(
 291                                     dbc.Row(  # Throughput
 292                                         id="row-graph-tput",
 293                                         class_name="g-0 p-2",
 294                                         children=[C.PLACEHOLDER, ]
 295                                     ),
 296                                     width=6
 297                                 ),
 298                                 dbc.Col(
 299                                     dbc.Row(  # Latency
 300                                         id="row-graph-lat",
 301                                         class_name="g-0 p-2",
 302                                         children=[C.PLACEHOLDER, ]
 303                                     ),
 304                                     width=6
 305                                 )
 306                             ]
 307                         ),
 308                         dbc.Row(  # Tables
 309                             id="row-table",
 310                             class_name="g-0 p-2",
 311                             children=[C.PLACEHOLDER, ]
 312                         ),
 313                         dbc.Row(  # Download
 314                             id="row-btn-download",
 315                             class_name="g-0 p-2",
 316                             children=[C.PLACEHOLDER, ]
 317                         )
 318                     ]
 319                 )
 320             ],
 321             width=9
 322         )
 323
 324     def _add_ctrl_panel(self) -> dbc.Row:
 325         """
 326         """
 327         return dbc.Row(
 328             id="row-ctrl-panel",
 329             class_name="g-0 p-2",
 330             children=[
 331                 dbc.Row(
 332                     class_name="g-0",
 333                     children=[
 334                         dbc.InputGroup(
 335                             [
 336                                 dbc.InputGroupText(
 337                                     children=show_tooltip(self._tooltips,
 338                                         "help-release", "CSIT Release")
 339                                 ),
 340                                 dbc.Select(
 341                                     id="dd-ctrl-rls",
 342                                     placeholder=("Select a Release..."),
 343                                     options=sorted(
 344                                         [
 345                                             {"label": k, "value": k} \
 346                                                 for k in self.spec_tbs.keys()
 347                                         ],
 348                                         key=lambda d: d["label"]
 349                                     )
 350                                 )
 351                             ],
 352                             class_name="mb-3",
 353                             size="sm",
 354                         ),
 355                     ]
 356                 ),
 357                 dbc.Row(
 358                     class_name="g-0",
 359                     children=[
 360                         dbc.InputGroup(
 361                             [
 362                                 dbc.InputGroupText(
 363                                     children=show_tooltip(self._tooltips,
 364                                         "help-dut", "DUT")
 365                                 ),
 366                                 dbc.Select(
 367                                     id="dd-ctrl-dut",
 368                                     placeholder=(
 369                                         "Select a Device under Test..."
 370                                     )
 371                                 )
 372                             ],
 373                             class_name="mb-3",
 374                             size="sm",
 375                         ),
 376                     ]
 377                 ),
 378                 dbc.Row(
 379                     class_name="g-0",
 380                     children=[
 381                         dbc.InputGroup(
 382                             [
 383                                 dbc.InputGroupText(
 384                                     children=show_tooltip(self._tooltips,
 385                                         "help-dut-ver", "DUT Version")
 386                                 ),
 387                                 dbc.Select(
 388                                     id="dd-ctrl-dutver",
 389                                     placeholder=(
 390                                         "Select a Version of "
 391                                         "Device under Test..."
 392                                     )
 393                                 )
 394                             ],
 395                             class_name="mb-3",
 396                             size="sm",
 397                         ),
 398                     ]
 399                 ),
 400                 dbc.Row(
 401                     class_name="g-0",
 402                     children=[
 403                         dbc.InputGroup(
 404                             [
 405                                 dbc.InputGroupText(
 406                                     children=show_tooltip(self._tooltips,
 407                                         "help-infra", "Infra")
 408                                 ),
 409                                 dbc.Select(
 410                                     id="dd-ctrl-phy",
 411                                     placeholder=(
 412                                         "Select a Physical Test Bed "
 413                                         "Topology..."
 414                                     )
 415                                 )
 416                             ],
 417                             class_name="mb-3",
 418                             size="sm",
 419                         ),
 420                     ]
 421                 ),
 422                 dbc.Row(
 423                     class_name="g-0",
 424                     children=[
 425                         dbc.InputGroup(
 426                             [
 427                                 dbc.InputGroupText(
 428                                     children=show_tooltip(self._tooltips,
 429                                         "help-area", "Area")
 430                                 ),
 431                                 dbc.Select(
 432                                     id="dd-ctrl-area",
 433                                     placeholder="Select an Area...",
 434                                     disabled=True,
 435                                 ),
 436                             ],
 437                             class_name="mb-3",
 438                             size="sm",
 439                         ),
 440                     ]
 441                 ),
 442                 dbc.Row(
 443                     class_name="g-0",
 444                     children=[
 445                         dbc.InputGroup(
 446                             [
 447                                 dbc.InputGroupText(
 448                                     children=show_tooltip(self._tooltips,
 449                                         "help-test", "Test")
 450                                 ),
 451                                 dbc.Select(
 452                                     id="dd-ctrl-test",
 453                                     placeholder="Select a Test...",
 454                                     disabled=True,
 455                                 ),
 456                             ],
 457                             class_name="mb-3",
 458                             size="sm",
 459                         ),
 460                     ]
 461                 ),
 462                 dbc.Row(
 463                     id="row-ctrl-framesize",
 464                     class_name="gy-1",
 465                     children=[
 466                         dbc.Label(
 467                             children=show_tooltip(self._tooltips,
 468                                 "help-framesize", "Frame Size"),
 469                             class_name="p-0"
 470                         ),
 471                         dbc.Col(
 472                             children=[
 473                                 dbc.Checklist(
 474                                     id="cl-ctrl-framesize-all",
 475                                     options=C.CL_ALL_DISABLED,
 476                                     inline=True,
 477                                     switch=False
 478                                 ),
 479                             ],
 480                             width=3
 481                         ),
 482                         dbc.Col(
 483                             children=[
 484                                 dbc.Checklist(
 485                                     id="cl-ctrl-framesize",
 486                                     inline=True,
 487                                     switch=False
 488                                 )
 489                             ]
 490                         )
 491                     ]
 492                 ),
 493                 dbc.Row(
 494                     id="row-ctrl-core",
 495                     class_name="gy-1",
 496                     children=[
 497                         dbc.Label(
 498                             children=show_tooltip(self._tooltips,
 499                                 "help-cores", "Number of Cores"),
 500                             class_name="p-0"
 501                         ),
 502                         dbc.Col(
 503                             children=[
 504                                 dbc.Checklist(
 505                                     id="cl-ctrl-core-all",
 506                                     options=C.CL_ALL_DISABLED,
 507                                     inline=False,
 508                                     switch=False
 509                                 )
 510                             ],
 511                             width=3
 512                         ),
 513                         dbc.Col(
 514                             children=[
 515                                 dbc.Checklist(
 516                                     id="cl-ctrl-core",
 517                                     inline=True,
 518                                     switch=False
 519                                 )
 520                             ]
 521                         )
 522                     ]
 523                 ),
 524                 dbc.Row(
 525                     id="row-ctrl-testtype",
 526                     class_name="gy-1",
 527                     children=[
 528                         dbc.Label(
 529                             children=show_tooltip(self._tooltips,
 530                                 "help-ttype", "Test Type"),
 531                             class_name="p-0"
 532                         ),
 533                         dbc.Col(
 534                             children=[
 535                                 dbc.Checklist(
 536                                     id="cl-ctrl-testtype-all",
 537                                     options=C.CL_ALL_DISABLED,
 538                                     inline=True,
 539                                     switch=False
 540                                 ),
 541                             ],
 542                             width=3
 543                         ),
 544                         dbc.Col(
 545                             children=[
 546                                 dbc.Checklist(
 547                                     id="cl-ctrl-testtype",
 548                                     inline=True,
 549                                     switch=False
 550                                 )
 551                             ]
 552                         )
 553                     ]
 554                 ),
 555                 dbc.Row(
 556                     id="row-ctrl-normalize",
 557                     class_name="gy-1",
 558                     children=[
 559                         dbc.Label(
 560                             children=show_tooltip(self._tooltips,
 561                                 "help-normalize", "Normalize"),
 562                             class_name="p-0"
 563                         ),
 564                         dbc.Col(
 565                             children=[
 566                                 dbc.Checklist(
 567                                     id="cl-ctrl-normalize",
 568                                     options=[{
 569                                         "value": "normalize",
 570                                         "label": (
 571                                             "Normalize results to CPU"
 572                                             "frequency 2GHz"
 573                                         )
 574                                     }],
 575                                     value=[],
 576                                     inline=True,
 577                                     switch=False
 578                                 ),
 579                             ]
 580                         )
 581                     ]
 582                 ),
 583                 dbc.Row(
 584                     class_name="gy-1 p-0",
 585                     children=[
 586                         dbc.ButtonGroup(
 587                             [
 588                                 dbc.Button(
 589                                     id="btn-ctrl-add",
 590                                     children="Add Selected",
 591                                     class_name="me-1",
 592                                     color="info"
 593                                 )
 594                             ],
 595                             size="md",
 596                         )
 597                     ]
 598                 ),
 599                 dbc.Row(
 600                     id="row-card-sel-tests",
 601                     class_name="gy-1",
 602                     style=C.STYLE_DISABLED,
 603                     children=[
 604                         dbc.Label(
 605                             "Selected tests",
 606                             class_name="p-0"
 607                         ),
 608                         dbc.Checklist(
 609                             class_name="overflow-auto",
 610                             id="cl-selected",
 611                             options=[],
 612                             inline=False,
 613                             style={"max-height": "20em"},
 614                         )
 615                     ],
 616                 ),
 617                 dbc.Row(
 618                     id="row-btns-sel-tests",
 619                     style=C.STYLE_DISABLED,
 620                     children=[
 621                         dbc.ButtonGroup(
 622                             class_name="gy-2",
 623                             children=[
 624                                 dbc.Button(
 625                                     id="btn-sel-remove",
 626                                     children="Remove Selected",
 627                                     class_name="w-100 me-1",
 628                                     color="info",
 629                                     disabled=False
 630                                 ),
 631                                 dbc.Button(
 632                                     id="btn-sel-remove-all",
 633                                     children="Remove All",
 634                                     class_name="w-100 me-1",
 635                                     color="info",
 636                                     disabled=False
 637                                 ),
 638                             ],
 639                             size="md",
 640                         )
 641                     ]
 642                 ),
 643             ]
 644         )
 645
 646     class ControlPanel:
 647         def __init__(self, panel: dict) -> None:
 648
 649             # Defines also the order of keys
 650             self._defaults = {
 651                 "dd-rls-value": str(),
 652                 "dd-dut-options": list(),
 653                 "dd-dut-disabled": True,
 654                 "dd-dut-value": str(),
 655                 "dd-dutver-options": list(),
 656                 "dd-dutver-disabled": True,
 657                 "dd-dutver-value": str(),
 658                 "dd-phy-options": list(),
 659                 "dd-phy-disabled": True,
 660                 "dd-phy-value": str(),
 661                 "dd-area-options": list(),
 662                 "dd-area-disabled": True,
 663                 "dd-area-value": str(),
 664                 "dd-test-options": list(),
 665                 "dd-test-disabled": True,
 666                 "dd-test-value": str(),
 667                 "cl-core-options": list(),
 668                 "cl-core-value": list(),
 669                 "cl-core-all-value": list(),
 670                 "cl-core-all-options": C.CL_ALL_DISABLED,
 671                 "cl-framesize-options": list(),
 672                 "cl-framesize-value": list(),
 673                 "cl-framesize-all-value": list(),
 674                 "cl-framesize-all-options": C.CL_ALL_DISABLED,
 675                 "cl-testtype-options": list(),
 676                 "cl-testtype-value": list(),
 677                 "cl-testtype-all-value": list(),
 678                 "cl-testtype-all-options": C.CL_ALL_DISABLED,
 679                 "btn-add-disabled": True,
 680                 "cl-normalize-value": list(),
 681                 "cl-selected-options": list()
 682             }
 683
 684             self._panel = deepcopy(self._defaults)
 685             if panel:
 686                 for key in self._defaults:
 687                     self._panel[key] = panel[key]
 688
 689         @property
 690         def defaults(self) -> dict:
 691             return self._defaults
 692
 693         @property
 694         def panel(self) -> dict:
 695             return self._panel
 696
 697         def set(self, kwargs: dict) -> None:
 698             for key, val in kwargs.items():
 699                 if key in self._panel:
 700                     self._panel[key] = val
 701                 else:
 702                     raise KeyError(f"The key {key} is not defined.")
 703
 704         def get(self, key: str) -> any:
 705             return self._panel[key]
 706
 707         def values(self) -> tuple:
 708             return tuple(self._panel.values())
 709
 710     def callbacks(self, app):
 711
 712         def _generate_plotting_area(figs: tuple, table: pd.DataFrame,
 713                 url: str) -> tuple:
 714             """
 715             """
 716
 717             (fig_tput, fig_lat) = figs
 718
 719             row_fig_tput = C.PLACEHOLDER
 720             row_fig_lat = C.PLACEHOLDER
 721             row_table = C.PLACEHOLDER
 722             row_btn_dwnld = C.PLACEHOLDER
 723
 724             if fig_tput:
 725                 row_fig_tput = [
 726                     dcc.Graph(
 727                         id={"type": "graph", "index": "tput"},
 728                         figure=fig_tput
 729                     )
 730                 ]
 731                 row_btn_dwnld = [
 732                     dbc.Col(  # Download
 733                         width=2,
 734                         children=[
 735                             dcc.Loading(children=[
 736                                 dbc.Button(
 737                                     id="btn-download-data",
 738                                     children=show_tooltip(self._tooltips,
 739                                         "help-download", "Download Data"),
 740                                     class_name="me-1",
 741                                     color="info"
 742                                 ),
 743                                 dcc.Download(id="download-data")
 744                             ]),
 745                         ]
 746                     ),
 747                     dbc.Col(  # Show URL
 748                         width=10,
 749                         children=[
 750                             dbc.InputGroup(
 751                                 class_name="me-1",
 752                                 children=[
 753                                     dbc.InputGroupText(
 754                                         style=C.URL_STYLE,
 755                                         children=show_tooltip(self._tooltips,
 756                                             "help-url", "URL", "input-url")
 757                                     ),
 758                                     dbc.Input(
 759                                         id="input-url",
 760                                         readonly=True,
 761                                         type="url",
 762                                         style=C.URL_STYLE,
 763                                         value=url
 764                                     )
 765                                 ]
 766                             )
 767                         ]
 768                     )
 769                 ]
 770             if fig_lat:
 771                 row_fig_lat = [
 772                     dcc.Graph(
 773                         id={"type": "graph", "index": "lat"},
 774                         figure=fig_lat
 775                     )
 776                 ]
 777             if not table.empty:
 778                 row_table = [
 779                     dbc.Table.from_dataframe(
 780                         table,
 781                         id={"type": "table", "index": "compare"},
 782                         striped=True,
 783                         bordered=True,
 784                         hover=True
 785                     )
 786                 ]
 787
 788             return row_fig_tput, row_fig_lat, row_table, row_btn_dwnld
 789
 790         @app.callback(
 791             Output("control-panel", "data"),  # Store
 792             Output("selected-tests", "data"),  # Store
 793             Output("row-graph-tput", "children"),
 794             Output("row-graph-lat", "children"),
 795             Output("row-table", "children"),
 796             Output("row-btn-download", "children"),
 797             Output("row-card-sel-tests", "style"),
 798             Output("row-btns-sel-tests", "style"),
 799             Output("dd-ctrl-rls", "value"),
 800             Output("dd-ctrl-dut", "options"),
 801             Output("dd-ctrl-dut", "disabled"),
 802             Output("dd-ctrl-dut", "value"),
 803             Output("dd-ctrl-dutver", "options"),
 804             Output("dd-ctrl-dutver", "disabled"),
 805             Output("dd-ctrl-dutver", "value"),
 806             Output("dd-ctrl-phy", "options"),
 807             Output("dd-ctrl-phy", "disabled"),
 808             Output("dd-ctrl-phy", "value"),
 809             Output("dd-ctrl-area", "options"),
 810             Output("dd-ctrl-area", "disabled"),
 811             Output("dd-ctrl-area", "value"),
 812             Output("dd-ctrl-test", "options"),
 813             Output("dd-ctrl-test", "disabled"),
 814             Output("dd-ctrl-test", "value"),
 815             Output("cl-ctrl-core", "options"),
 816             Output("cl-ctrl-core", "value"),
 817             Output("cl-ctrl-core-all", "value"),
 818             Output("cl-ctrl-core-all", "options"),
 819             Output("cl-ctrl-framesize", "options"),
 820             Output("cl-ctrl-framesize", "value"),
 821             Output("cl-ctrl-framesize-all", "value"),
 822             Output("cl-ctrl-framesize-all", "options"),
 823             Output("cl-ctrl-testtype", "options"),
 824             Output("cl-ctrl-testtype", "value"),
 825             Output("cl-ctrl-testtype-all", "value"),
 826             Output("cl-ctrl-testtype-all", "options"),
 827             Output("btn-ctrl-add", "disabled"),
 828             Output("cl-ctrl-normalize", "value"),
 829             Output("cl-selected", "options"),  # User selection
 830             State("control-panel", "data"),  # Store
 831             State("selected-tests", "data"),  # Store
 832             State("cl-selected", "value"),  # User selection
 833             Input("dd-ctrl-rls", "value"),
 834             Input("dd-ctrl-dut", "value"),
 835             Input("dd-ctrl-dutver", "value"),
 836             Input("dd-ctrl-phy", "value"),
 837             Input("dd-ctrl-area", "value"),
 838             Input("dd-ctrl-test", "value"),
 839             Input("cl-ctrl-core", "value"),
 840             Input("cl-ctrl-core-all", "value"),
 841             Input("cl-ctrl-framesize", "value"),
 842             Input("cl-ctrl-framesize-all", "value"),
 843             Input("cl-ctrl-testtype", "value"),
 844             Input("cl-ctrl-testtype-all", "value"),
 845             Input("cl-ctrl-normalize", "value"),
 846             Input("btn-ctrl-add", "n_clicks"),
 847             Input("btn-sel-remove", "n_clicks"),
 848             Input("btn-sel-remove-all", "n_clicks"),
 849             Input("url", "href")
 850         )
 851         def _update_ctrl_panel(cp_data: dict, store_sel: list, list_sel: list,
 852             dd_rls: str, dd_dut: str, dd_dutver: str, dd_phy: str, dd_area: str,
 853             dd_test: str, cl_core: list, cl_core_all: list, cl_framesize: list,
 854             cl_framesize_all: list, cl_testtype: list, cl_testtype_all: list,
 855             cl_normalize: list, btn_add: int, btn_remove: int,
 856             btn_remove_all: int, href: str) -> tuple:
 857             """
 858             """
 859
 860             ctrl_panel = self.ControlPanel(cp_data)
 861
 862             # Parse the url:
 863             parsed_url = url_decode(href)
 864
 865             row_fig_tput = no_update
 866             row_fig_lat = no_update
 867             row_table = no_update
 868             row_btn_dwnld = no_update
 869             row_card_sel_tests = no_update
 870             row_btns_sel_tests = no_update
 871
 872             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 873
 874             if trigger_id == "dd-ctrl-rls":
 875                 try:
 876                     rls = self.spec_tbs[dd_rls]
 877                     options = sorted(
 878                         [{"label": v, "value": v} for v in rls.keys()],
 879                         key=lambda d: d["label"]
 880                     )
 881                     disabled = False
 882                 except KeyError:
 883                     options = list()
 884                     disabled = True
 885                 ctrl_panel.set({
 886                     "dd-rls-value": dd_rls,
 887                     "dd-dut-value": str(),
 888                     "dd-dut-options": options,
 889                     "dd-dut-disabled": disabled,
 890                     "dd-dutver-value": str(),
 891                     "dd-dutver-options": list(),
 892                     "dd-dutver-disabled": True,
 893                     "dd-phy-value": str(),
 894                     "dd-phy-options": list(),
 895                     "dd-phy-disabled": True,
 896                     "dd-area-value": str(),
 897                     "dd-area-options": list(),
 898                     "dd-area-disabled": True,
 899                     "dd-test-value": str(),
 900                     "dd-test-options": list(),
 901                     "dd-test-disabled": True,
 902                     "cl-core-options": list(),
 903                     "cl-core-value": list(),
 904                     "cl-core-all-value": list(),
 905                     "cl-core-all-options": C.CL_ALL_DISABLED,
 906                     "cl-framesize-options": list(),
 907                     "cl-framesize-value": list(),
 908                     "cl-framesize-all-value": list(),
 909                     "cl-framesize-all-options": C.CL_ALL_DISABLED,
 910                     "cl-testtype-options": list(),
 911                     "cl-testtype-value": list(),
 912                     "cl-testtype-all-value": list(),
 913                     "cl-testtype-all-options": C.CL_ALL_DISABLED
 914                 })
 915             elif trigger_id == "dd-ctrl-dut":
 916                 try:
 917                     rls = ctrl_panel.get("dd-rls-value")
 918                     dut = self.spec_tbs[rls][dd_dut]
 919                     options = sorted(
 920                         [{"label": v, "value": v} for v in dut.keys()],
 921                         key=lambda d: d["label"]
 922                     )
 923                     disabled = False
 924                 except KeyError:
 925                     options = list()
 926                     disabled = True
 927                 ctrl_panel.set({
 928                     "dd-dut-value": dd_dut,
 929                     "dd-dutver-value": str(),
 930                     "dd-dutver-options": options,
 931                     "dd-dutver-disabled": disabled,
 932                     "dd-phy-value": str(),
 933                     "dd-phy-options": list(),
 934                     "dd-phy-disabled": True,
 935                     "dd-area-value": str(),
 936                     "dd-area-options": list(),
 937                     "dd-area-disabled": True,
 938                     "dd-test-value": str(),
 939                     "dd-test-options": list(),
 940                     "dd-test-disabled": True,
 941                     "cl-core-options": list(),
 942                     "cl-core-value": list(),
 943                     "cl-core-all-value": list(),
 944                     "cl-core-all-options": C.CL_ALL_DISABLED,
 945                     "cl-framesize-options": list(),
 946                     "cl-framesize-value": list(),
 947                     "cl-framesize-all-value": list(),
 948                     "cl-framesize-all-options": C.CL_ALL_DISABLED,
 949                     "cl-testtype-options": list(),
 950                     "cl-testtype-value": list(),
 951                     "cl-testtype-all-value": list(),
 952                     "cl-testtype-all-options": C.CL_ALL_DISABLED
 953                 })
 954             elif trigger_id == "dd-ctrl-dutver":
 955                 try:
 956                     rls = ctrl_panel.get("dd-rls-value")
 957                     dut = ctrl_panel.get("dd-dut-value")
 958                     dutver = self.spec_tbs[rls][dut][dd_dutver]
 959                     options = sorted(
 960                         [{"label": v, "value": v} for v in dutver.keys()],
 961                         key=lambda d: d["label"]
 962                     )
 963                     disabled = False
 964                 except KeyError:
 965                     options = list()
 966                     disabled = True
 967                 ctrl_panel.set({
 968                     "dd-dutver-value": dd_dutver,
 969                     "dd-phy-value": str(),
 970                     "dd-phy-options": options,
 971                     "dd-phy-disabled": disabled,
 972                     "dd-area-value": str(),
 973                     "dd-area-options": list(),
 974                     "dd-area-disabled": True,
 975                     "dd-test-value": str(),
 976                     "dd-test-options": list(),
 977                     "dd-test-disabled": True,
 978                     "cl-core-options": list(),
 979                     "cl-core-value": list(),
 980                     "cl-core-all-value": list(),
 981                     "cl-core-all-options": C.CL_ALL_DISABLED,
 982                     "cl-framesize-options": list(),
 983                     "cl-framesize-value": list(),
 984                     "cl-framesize-all-value": list(),
 985                     "cl-framesize-all-options": C.CL_ALL_DISABLED,
 986                     "cl-testtype-options": list(),
 987                     "cl-testtype-value": list(),
 988                     "cl-testtype-all-value": list(),
 989                     "cl-testtype-all-options": C.CL_ALL_DISABLED
 990                 })
 991             elif trigger_id == "dd-ctrl-phy":
 992                 try:
 993                     rls = ctrl_panel.get("dd-rls-value")
 994                     dut = ctrl_panel.get("dd-dut-value")
 995                     dutver = ctrl_panel.get("dd-dutver-value")
 996                     phy = self.spec_tbs[rls][dut][dutver][dd_phy]
 997                     options = sorted(
 998                         [{"label": label(v), "value": v} for v in phy.keys()],
 999                         key=lambda d: d["label"]
1000                     )
1001                     disabled = False
1002                 except KeyError:
1003                     options = list()
1004                     disabled = True
1005                 ctrl_panel.set({
1006                     "dd-phy-value": dd_phy,
1007                     "dd-area-value": str(),
1008                     "dd-area-options": options,
1009                     "dd-area-disabled": disabled,
1010                     "dd-test-value": str(),
1011                     "dd-test-options": list(),
1012                     "dd-test-disabled": True,
1013                     "cl-core-options": list(),
1014                     "cl-core-value": list(),
1015                     "cl-core-all-value": list(),
1016                     "cl-core-all-options": C.CL_ALL_DISABLED,
1017                     "cl-framesize-options": list(),
1018                     "cl-framesize-value": list(),
1019                     "cl-framesize-all-value": list(),
1020                     "cl-framesize-all-options": C.CL_ALL_DISABLED,
1021                     "cl-testtype-options": list(),
1022                     "cl-testtype-value": list(),
1023                     "cl-testtype-all-value": list(),
1024                     "cl-testtype-all-options": C.CL_ALL_DISABLED
1025                 })
1026             elif trigger_id == "dd-ctrl-area":
1027                 try:
1028                     rls = ctrl_panel.get("dd-rls-value")
1029                     dut = ctrl_panel.get("dd-dut-value")
1030                     dutver = ctrl_panel.get("dd-dutver-value")
1031                     phy = ctrl_panel.get("dd-phy-value")
1032                     area = self.spec_tbs[rls][dut][dutver][phy][dd_area]
1033                     options = sorted(
1034                         [{"label": v, "value": v} for v in area.keys()],
1035                         key=lambda d: d["label"]
1036                     )
1037                     disabled = False
1038                 except KeyError:
1039                     options = list()
1040                     disabled = True
1041                 ctrl_panel.set({
1042                     "dd-area-value": dd_area,
1043                     "dd-test-value": str(),
1044                     "dd-test-options": options,
1045                     "dd-test-disabled": disabled,
1046                     "cl-core-options": list(),
1047                     "cl-core-value": list(),
1048                     "cl-core-all-value": list(),
1049                     "cl-core-all-options": C.CL_ALL_DISABLED,
1050                     "cl-framesize-options": list(),
1051                     "cl-framesize-value": list(),
1052                     "cl-framesize-all-value": list(),
1053                     "cl-framesize-all-options": C.CL_ALL_DISABLED,
1054                     "cl-testtype-options": list(),
1055                     "cl-testtype-value": list(),
1056                     "cl-testtype-all-value": list(),
1057                     "cl-testtype-all-options": C.CL_ALL_DISABLED
1058                 })
1059             elif trigger_id == "dd-ctrl-test":
1060                 rls = ctrl_panel.get("dd-rls-value")
1061                 dut = ctrl_panel.get("dd-dut-value")
1062                 dutver = ctrl_panel.get("dd-dutver-value")
1063                 phy = ctrl_panel.get("dd-phy-value")
1064                 area = ctrl_panel.get("dd-area-value")
1065                 test = self.spec_tbs[rls][dut][dutver][phy][area][dd_test]
1066                 if dut and phy and area and dd_test:
1067                     ctrl_panel.set({
1068                         "dd-test-value": dd_test,
1069                         "cl-core-options": [{"label": v, "value": v}
1070                             for v in sorted(test["core"])],
1071                         "cl-core-value": list(),
1072                         "cl-core-all-value": list(),
1073                         "cl-core-all-options": C.CL_ALL_ENABLED,
1074                         "cl-framesize-options": [{"label": v, "value": v}
1075                             for v in sorted(test["frame-size"])],
1076                         "cl-framesize-value": list(),
1077                         "cl-framesize-all-value": list(),
1078                         "cl-framesize-all-options": C.CL_ALL_ENABLED,
1079                         "cl-testtype-options": [{"label": v, "value": v}
1080                             for v in sorted(test["test-type"])],
1081                         "cl-testtype-value": list(),
1082                         "cl-testtype-all-value": list(),
1083                         "cl-testtype-all-options": C.CL_ALL_ENABLED,
1084                     })
1085             elif trigger_id == "cl-ctrl-core":
1086                 val_sel, val_all = sync_checklists(
1087                     options=ctrl_panel.get("cl-core-options"),
1088                     sel=cl_core,
1089                     all=list(),
1090                     id=""
1091                 )
1092                 ctrl_panel.set({
1093                     "cl-core-value": val_sel,
1094                     "cl-core-all-value": val_all,
1095                 })
1096             elif trigger_id == "cl-ctrl-core-all":
1097                 val_sel, val_all = sync_checklists(
1098                     options = ctrl_panel.get("cl-core-options"),
1099                     sel=list(),
1100                     all=cl_core_all,
1101                     id="all"
1102                 )
1103                 ctrl_panel.set({
1104                     "cl-core-value": val_sel,
1105                     "cl-core-all-value": val_all,
1106                 })
1107             elif trigger_id == "cl-ctrl-framesize":
1108                 val_sel, val_all = sync_checklists(
1109                     options = ctrl_panel.get("cl-framesize-options"),
1110                     sel=cl_framesize,
1111                     all=list(),
1112                     id=""
1113                 )
1114                 ctrl_panel.set({
1115                     "cl-framesize-value": val_sel,
1116                     "cl-framesize-all-value": val_all,
1117                 })
1118             elif trigger_id == "cl-ctrl-framesize-all":
1119                 val_sel, val_all = sync_checklists(
1120                     options = ctrl_panel.get("cl-framesize-options"),
1121                     sel=list(),
1122                     all=cl_framesize_all,
1123                     id="all"
1124                 )
1125                 ctrl_panel.set({
1126                     "cl-framesize-value": val_sel,
1127                     "cl-framesize-all-value": val_all,
1128                 })
1129             elif trigger_id == "cl-ctrl-testtype":
1130                 val_sel, val_all = sync_checklists(
1131                     options = ctrl_panel.get("cl-testtype-options"),
1132                     sel=cl_testtype,
1133                     all=list(),
1134                     id=""
1135                 )
1136                 ctrl_panel.set({
1137                     "cl-testtype-value": val_sel,
1138                     "cl-testtype-all-value": val_all,
1139                 })
1140             elif trigger_id == "cl-ctrl-testtype-all":
1141                 val_sel, val_all = sync_checklists(
1142                     options = ctrl_panel.get("cl-testtype-options"),
1143                     sel=list(),
1144                     all=cl_testtype_all,
1145                     id="all"
1146                 )
1147                 ctrl_panel.set({
1148                     "cl-testtype-value": val_sel,
1149                     "cl-testtype-all-value": val_all,
1150                 })
1151             elif trigger_id == "btn-ctrl-add":
1152                 _ = btn_add
1153                 rls = ctrl_panel.get("dd-rls-value")
1154                 dut = ctrl_panel.get("dd-dut-value")
1155                 dutver = ctrl_panel.get("dd-dutver-value")
1156                 phy = ctrl_panel.get("dd-phy-value")
1157                 area = ctrl_panel.get("dd-area-value")
1158                 test = ctrl_panel.get("dd-test-value")
1159                 cores = ctrl_panel.get("cl-core-value")
1160                 framesizes = ctrl_panel.get("cl-framesize-value")
1161                 testtypes = ctrl_panel.get("cl-testtype-value")
1162                 # Add selected test to the list of tests in store:
1163                 if all((rls, dut, dutver, phy, area, test, cores, framesizes,
1164                         testtypes)):
1165                     if store_sel is None:
1166                         store_sel = list()
1167                     for core in cores:
1168                         for framesize in framesizes:
1169                             for ttype in testtypes:
1170                                 if dut == "trex":
1171                                     core = str()
1172                                 tid = "-".join((rls, dut, dutver,
1173                                     phy.replace('af_xdp', 'af-xdp'), area,
1174                                     framesize.lower(), core.lower(), test,
1175                                     ttype.lower()))
1176                                 if tid not in [itm["id"] for itm in store_sel]:
1177                                     store_sel.append({
1178                                         "id": tid,
1179                                         "rls": rls,
1180                                         "dut": dut,
1181                                         "dutver": dutver,
1182                                         "phy": phy,
1183                                         "area": area,
1184                                         "test": test,
1185                                         "framesize": framesize.lower(),
1186                                         "core": core.lower(),
1187                                         "testtype": ttype.lower()
1188                                     })
1189                     store_sel = sorted(store_sel, key=lambda d: d["id"])
1190                     row_card_sel_tests = C.STYLE_ENABLED
1191                     row_btns_sel_tests = C.STYLE_ENABLED
1192                     if C.CLEAR_ALL_INPUTS:
1193                         ctrl_panel.set(ctrl_panel.defaults)
1194                     ctrl_panel.set({
1195                         "cl-selected-options": list_tests(store_sel)
1196                     })
1197             elif trigger_id == "btn-sel-remove-all":
1198                 _ = btn_remove_all
1199                 row_fig_tput = C.PLACEHOLDER
1200                 row_fig_lat = C.PLACEHOLDER
1201                 row_table = C.PLACEHOLDER
1202                 row_btn_dwnld = C.PLACEHOLDER
1203                 row_card_sel_tests = C.STYLE_DISABLED
1204                 row_btns_sel_tests = C.STYLE_DISABLED
1205                 store_sel = list()
1206                 ctrl_panel.set({"cl-selected-options": list()})
1207             elif trigger_id == "btn-sel-remove":
1208                 _ = btn_remove
1209                 if list_sel:
1210                     new_store_sel = list()
1211                     for item in store_sel:
1212                         if item["id"] not in list_sel:
1213                             new_store_sel.append(item)
1214                     store_sel = new_store_sel
1215             elif trigger_id == "url":
1216                 # TODO: Add verification
1217                 url_params = parsed_url["params"]
1218                 if url_params:
1219                     store_sel = literal_eval(
1220                         url_params.get("store_sel", list())[0])
1221                     if store_sel:
1222                         row_card_sel_tests = C.STYLE_ENABLED
1223                         row_btns_sel_tests = C.STYLE_ENABLED
1224
1225             if trigger_id in ("btn-ctrl-add", "url", "btn-sel-remove",
1226                     "cl-ctrl-normalize"):
1227                 if store_sel:
1228                     row_fig_tput, row_fig_lat, row_table, row_btn_dwnld = \
1229                         _generate_plotting_area(
1230                             graph_iterative(
1231                                 self.data, store_sel, self.layout,
1232                                 bool(cl_normalize)
1233                             ),
1234                             table_comparison(
1235                                 self.data, store_sel, bool(cl_normalize)
1236                             ),
1237                             gen_new_url(parsed_url, {"store_sel": store_sel})
1238                         )
1239                     ctrl_panel.set({
1240                         "cl-selected-options": list_tests(store_sel)
1241                     })
1242                 else:
1243                     row_fig_tput = C.PLACEHOLDER
1244                     row_fig_lat = C.PLACEHOLDER
1245                     row_table = C.PLACEHOLDER
1246                     row_btn_dwnld = C.PLACEHOLDER
1247                     row_card_sel_tests = C.STYLE_DISABLED
1248                     row_btns_sel_tests = C.STYLE_DISABLED
1249                     store_sel = list()
1250                     ctrl_panel.set({"cl-selected-options": list()})
1251
1252             if ctrl_panel.get("cl-core-value") and \
1253                     ctrl_panel.get("cl-framesize-value") and \
1254                     ctrl_panel.get("cl-testtype-value"):
1255                 disabled = False
1256             else:
1257                 disabled = True
1258             ctrl_panel.set({
1259                 "btn-add-disabled": disabled,
1260                 "cl-normalize-value": cl_normalize
1261             })
1262
1263             ret_val = [
1264                 ctrl_panel.panel, store_sel,
1265                 row_fig_tput, row_fig_lat, row_table, row_btn_dwnld,
1266                 row_card_sel_tests, row_btns_sel_tests
1267             ]
1268             ret_val.extend(ctrl_panel.values())
1269             return ret_val
1270
1271         # @app.callback(
1272         #     Output("download-data", "data"),
1273         #     State("selected-tests", "data"),
1274         #     Input("btn-download-data", "n_clicks"),
1275         #     prevent_initial_call=True
1276         # )
1277         # def _download_data(store_sel, n_clicks):
1278         #     """
1279         #     """
1280
1281         #     if not n_clicks:
1282         #         raise PreventUpdate
1283
1284         #     if not store_sel:
1285         #         raise PreventUpdate
1286
1287         #     df = pd.DataFrame()
1288         #     for itm in store_sel:
1289         #         sel_data = select_trending_data(self.data, itm)
1290         #         if sel_data is None:
1291         #             continue
1292         #         df = pd.concat([df, sel_data], ignore_index=True)
1293
1294         #     return dcc.send_data_frame(df.to_csv, "trending_data.csv")