csit.infra.dash/app/cdash/coverage/layout.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17
  18 import pandas as pd
  19 import dash_bootstrap_components as dbc
  20
  21 from flask import Flask
  22 from dash import dcc
  23 from dash import html
  24 from dash import callback_context, no_update, ALL
  25 from dash import Input, Output, State
  26 from dash.exceptions import PreventUpdate
  27 from ast import literal_eval
  28
  29 from ..utils.constants import Constants as C
  30 from ..utils.control_panel import ControlPanel
  31 from ..utils.trigger import Trigger
  32 from ..utils.utils import label, gen_new_url, generate_options
  33 from ..utils.url_processing import url_decode
  34 from .tables import coverage_tables, select_coverage_data
  35
  36
  37 # Control panel partameters and their default values.
  38 CP_PARAMS = {
  39     "rls-val": str(),
  40     "dut-opt": list(),
  41     "dut-dis": True,
  42     "dut-val": str(),
  43     "dutver-opt": list(),
  44     "dutver-dis": True,
  45     "dutver-val": str(),
  46     "phy-opt": list(),
  47     "phy-dis": True,
  48     "phy-val": str(),
  49     "area-opt": list(),
  50     "area-dis": True,
  51     "area-val": str()
  52 }
  53
  54
  55 class Layout:
  56     """The layout of the dash app and the callbacks.
  57     """
  58
  59     def __init__(
  60             self,
  61             app: Flask,
  62             data_coverage: pd.DataFrame,
  63             html_layout_file: str
  64         ) -> None:
  65         """Initialization:
  66         - save the input parameters,
  67         - prepare data for the control panel,
  68         - read HTML layout file,
  69
  70         :param app: Flask application running the dash application.
  71         :param html_layout_file: Path and name of the file specifying the HTML
  72             layout of the dash application.
  73         :type app: Flask
  74         :type html_layout_file: str
  75         """
  76
  77         # Inputs
  78         self._app = app
  79         self._html_layout_file = html_layout_file
  80         self._data = data_coverage
  81
  82         # Get structure of tests:
  83         tbs = dict()
  84         cols = ["job", "test_id", "dut_version", "release", ]
  85         for _, row in self._data[cols].drop_duplicates().iterrows():
  86             rls = row["release"]
  87             lst_job = row["job"].split("-")
  88             dut = lst_job[1]
  89             d_ver = row["dut_version"]
  90             tbed = "-".join(lst_job[-2:])
  91             lst_test_id = row["test_id"].split(".")
  92             if dut == "dpdk":
  93                 area = "dpdk"
  94             else:
  95                 area = ".".join(lst_test_id[3:-2])
  96             suite = lst_test_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
  97                 replace("2n-", "")
  98             test = lst_test_id[-1]
  99             nic = suite.split("-")[0]
 100             for drv in C.DRIVERS:
 101                 if drv in test:
 102                     driver = drv.replace("-", "_")
 103                     test = test.replace(f"{drv}-", "")
 104                     break
 105             else:
 106                 driver = "dpdk"
 107             infra = "-".join((tbed, nic, driver))
 108
 109             if tbs.get(rls, None) is None:
 110                 tbs[rls] = dict()
 111             if tbs[rls].get(dut, None) is None:
 112                 tbs[rls][dut] = dict()
 113             if tbs[rls][dut].get(d_ver, None) is None:
 114                 tbs[rls][dut][d_ver] = dict()
 115             if tbs[rls][dut][d_ver].get(infra, None) is None:
 116                 tbs[rls][dut][d_ver][infra] = list()
 117             if area not in tbs[rls][dut][d_ver][infra]:
 118                 tbs[rls][dut][d_ver][infra].append(area)
 119
 120         self._spec_tbs = tbs
 121
 122         # Read from files:
 123         self._html_layout = str()
 124
 125         try:
 126             with open(self._html_layout_file, "r") as file_read:
 127                 self._html_layout = file_read.read()
 128         except IOError as err:
 129             raise RuntimeError(
 130                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 131             )
 132
 133         # Callbacks:
 134         if self._app is not None and hasattr(self, "callbacks"):
 135             self.callbacks(self._app)
 136
 137     @property
 138     def html_layout(self):
 139         return self._html_layout
 140
 141     def add_content(self):
 142         """Top level method which generated the web page.
 143
 144         It generates:
 145         - Store for user input data,
 146         - Navigation bar,
 147         - Main area with control panel and ploting area.
 148
 149         If no HTML layout is provided, an error message is displayed instead.
 150
 151         :returns: The HTML div with the whole page.
 152         :rtype: html.Div
 153         """
 154
 155         if self.html_layout and self._spec_tbs:
 156             return html.Div(
 157                 id="div-main",
 158                 className="small",
 159                 children=[
 160                     dbc.Row(
 161                         id="row-navbar",
 162                         class_name="g-0",
 163                         children=[
 164                             self._add_navbar()
 165                         ]
 166                     ),
 167                     dbc.Row(
 168                         id="row-main",
 169                         class_name="g-0",
 170                         children=[
 171                             dcc.Store(id="store-selected-tests"),
 172                             dcc.Store(id="store-control-panel"),
 173                             dcc.Location(id="url", refresh=False),
 174                             self._add_ctrl_col(),
 175                             self._add_plotting_col()
 176                         ]
 177                     )
 178                 ]
 179             )
 180         else:
 181             return html.Div(
 182                 id="div-main-error",
 183                 children=[
 184                     dbc.Alert(
 185                         [
 186                             "An Error Occured"
 187                         ],
 188                         color="danger"
 189                     )
 190                 ]
 191             )
 192
 193     def _add_navbar(self):
 194         """Add nav element with navigation panel. It is placed on the top.
 195
 196         :returns: Navigation bar.
 197         :rtype: dbc.NavbarSimple
 198         """
 199         return dbc.NavbarSimple(
 200             id="navbarsimple-main",
 201             children=[
 202                 dbc.NavItem(
 203                     dbc.NavLink(
 204                         C.COVERAGE_TITLE,
 205                         disabled=True,
 206                         external_link=True,
 207                         href="#"
 208                     )
 209                 )
 210             ],
 211             brand=C.BRAND,
 212             brand_href="/",
 213             brand_external_link=True,
 214             class_name="p-2",
 215             fluid=True
 216         )
 217
 218     def _add_ctrl_col(self) -> dbc.Col:
 219         """Add column with controls. It is placed on the left side.
 220
 221         :returns: Column with the control panel.
 222         :rtype: dbc.Col
 223         """
 224         return dbc.Col([
 225             html.Div(
 226                 children=self._add_ctrl_panel(),
 227                 className="sticky-top"
 228             )
 229         ])
 230
 231     def _add_plotting_col(self) -> dbc.Col:
 232         """Add column with plots. It is placed on the right side.
 233
 234         :returns: Column with plots.
 235         :rtype: dbc.Col
 236         """
 237         return dbc.Col(
 238             id="col-plotting-area",
 239             children=[
 240                 dbc.Spinner(
 241                     children=[
 242                         dbc.Row(
 243                             id="plotting-area",
 244                             class_name="g-0 p-0",
 245                             children=[
 246                                 C.PLACEHOLDER
 247                             ]
 248                         )
 249                     ]
 250                 )
 251             ],
 252             width=9
 253         )
 254
 255     def _add_ctrl_panel(self) -> list:
 256         """Add control panel.
 257
 258         :returns: Control panel.
 259         :rtype: list
 260         """
 261         return [
 262             dbc.Row(
 263                 class_name="g-0 p-1",
 264                 children=[
 265                     dbc.InputGroup(
 266                         [
 267                             dbc.InputGroupText("CSIT Release"),
 268                             dbc.Select(
 269                                 id={"type": "ctrl-dd", "index": "rls"},
 270                                 placeholder="Select a Release...",
 271                                 options=sorted(
 272                                     [
 273                                         {"label": k, "value": k} \
 274                                             for k in self._spec_tbs.keys()
 275                                     ],
 276                                     key=lambda d: d["label"]
 277                                 )
 278                             )
 279                         ],
 280                         size="sm"
 281                     )
 282                 ]
 283             ),
 284             dbc.Row(
 285                 class_name="g-0 p-1",
 286                 children=[
 287                     dbc.InputGroup(
 288                         [
 289                             dbc.InputGroupText("DUT"),
 290                             dbc.Select(
 291                                 id={"type": "ctrl-dd", "index": "dut"},
 292                                 placeholder="Select a Device under Test..."
 293                             )
 294                         ],
 295                         size="sm"
 296                     )
 297                 ]
 298             ),
 299             dbc.Row(
 300                 class_name="g-0 p-1",
 301                 children=[
 302                     dbc.InputGroup(
 303                         [
 304                             dbc.InputGroupText("DUT Version"),
 305                             dbc.Select(
 306                                 id={"type": "ctrl-dd", "index": "dutver"},
 307                                 placeholder=\
 308                                     "Select a Version of Device under Test..."
 309                             )
 310                         ],
 311                         size="sm"
 312                     )
 313                 ]
 314             ),
 315             dbc.Row(
 316                 class_name="g-0 p-1",
 317                 children=[
 318                     dbc.InputGroup(
 319                         [
 320                             dbc.InputGroupText("Infra"),
 321                             dbc.Select(
 322                                 id={"type": "ctrl-dd", "index": "phy"},
 323                                 placeholder=\
 324                                     "Select a Physical Test Bed Topology..."
 325                             )
 326                         ],
 327                         size="sm"
 328                     )
 329                 ]
 330             ),
 331             dbc.Row(
 332                 class_name="g-0 p-1",
 333                 children=[
 334                     dbc.InputGroup(
 335                         [
 336                             dbc.InputGroupText("Area"),
 337                             dbc.Select(
 338                                 id={"type": "ctrl-dd", "index": "area"},
 339                                 placeholder="Select an Area..."
 340                             )
 341                         ],
 342                         size="sm"
 343                     )
 344                 ]
 345             )
 346         ]
 347
 348     def _get_plotting_area(self, selected: dict, url: str) -> list:
 349         """Generate the plotting area with all its content.
 350
 351         :param selected: Selected parameters of tests.
 352         :param url: URL to be displayed in the modal window.
 353         :type selected: dict
 354         :type url: str
 355         :returns: List of rows with elements to be displayed in the plotting
 356             area.
 357         :rtype: list
 358         """
 359         if not selected:
 360             return C.PLACEHOLDER
 361
 362         return [
 363             dbc.Row(
 364                 children=coverage_tables(self._data, selected),
 365                 class_name="g-0 p-0",
 366             ),
 367             dbc.Row(
 368                 children=C.PLACEHOLDER,
 369                 class_name="g-0 p-1"
 370             ),
 371             dbc.Row(
 372                 [
 373                     dbc.Col([html.Div(
 374                         [
 375                             dbc.Button(
 376                                 id="plot-btn-url",
 377                                 children="Show URL",
 378                                 class_name="me-1",
 379                                 color="info",
 380                                 style={
 381                                     "text-transform": "none",
 382                                     "padding": "0rem 1rem"
 383                                 }
 384                             ),
 385                             dbc.Modal(
 386                                 [
 387                                     dbc.ModalHeader(dbc.ModalTitle("URL")),
 388                                     dbc.ModalBody(url)
 389                                 ],
 390                                 id="plot-mod-url",
 391                                 size="xl",
 392                                 is_open=False,
 393                                 scrollable=True
 394                             ),
 395                             dbc.Button(
 396                                 id="plot-btn-download",
 397                                 children="Download Data",
 398                                 class_name="me-1",
 399                                 color="info",
 400                                 style={
 401                                     "text-transform": "none",
 402                                     "padding": "0rem 1rem"
 403                                 }
 404                             ),
 405                             dcc.Download(id="download-iterative-data")
 406                         ],
 407                         className=\
 408                             "d-grid gap-0 d-md-flex justify-content-md-end"
 409                     )])
 410                 ],
 411                 class_name="g-0 p-0"
 412             ),
 413             dbc.Row(
 414                 children=C.PLACEHOLDER,
 415                 class_name="g-0 p-1"
 416             )
 417         ]
 418
 419     def callbacks(self, app):
 420         """Callbacks for the whole application.
 421
 422         :param app: The application.
 423         :type app: Flask
 424         """
 425
 426         @app.callback(
 427             [
 428                 Output("store-control-panel", "data"),
 429                 Output("store-selected-tests", "data"),
 430                 Output("plotting-area", "children"),
 431                 Output({"type": "ctrl-dd", "index": "rls"}, "value"),
 432                 Output({"type": "ctrl-dd", "index": "dut"}, "options"),
 433                 Output({"type": "ctrl-dd", "index": "dut"}, "disabled"),
 434                 Output({"type": "ctrl-dd", "index": "dut"}, "value"),
 435                 Output({"type": "ctrl-dd", "index": "dutver"}, "options"),
 436                 Output({"type": "ctrl-dd", "index": "dutver"}, "disabled"),
 437                 Output({"type": "ctrl-dd", "index": "dutver"}, "value"),
 438                 Output({"type": "ctrl-dd", "index": "phy"}, "options"),
 439                 Output({"type": "ctrl-dd", "index": "phy"}, "disabled"),
 440                 Output({"type": "ctrl-dd", "index": "phy"}, "value"),
 441                 Output({"type": "ctrl-dd", "index": "area"}, "options"),
 442                 Output({"type": "ctrl-dd", "index": "area"}, "disabled"),
 443                 Output({"type": "ctrl-dd", "index": "area"}, "value"),
 444             ],
 445             [
 446                 State("store-control-panel", "data"),
 447                 State("store-selected-tests", "data")
 448             ],
 449             [
 450                 Input("url", "href"),
 451                 Input({"type": "ctrl-dd", "index": ALL}, "value")
 452             ]
 453         )
 454         def _update_application(
 455                 control_panel: dict,
 456                 selected: dict,
 457                 href: str,
 458                 *_
 459             ) -> tuple:
 460             """Update the application when the event is detected.
 461             """
 462
 463             ctrl_panel = ControlPanel(CP_PARAMS, control_panel)
 464             plotting_area = no_update
 465             on_draw = False
 466             if selected is None:
 467                 selected = dict()
 468
 469             # Parse the url:
 470             parsed_url = url_decode(href)
 471             if parsed_url:
 472                 url_params = parsed_url["params"]
 473             else:
 474                 url_params = None
 475
 476             trigger = Trigger(callback_context.triggered)
 477
 478             if trigger.type == "url" and url_params:
 479                 try:
 480                     selected = literal_eval(url_params["selection"][0])
 481                 except (KeyError, IndexError):
 482                     pass
 483                 if selected:
 484                     ctrl_panel.set({
 485                         "rls-val": selected["rls"],
 486                         "dut-val": selected["dut"],
 487                         "dut-opt": generate_options(
 488                             self._spec_tbs[selected["rls"]].keys()
 489                         ),
 490                         "dut-dis": False,
 491                         "dutver-val": selected["dutver"],
 492                         "dutver-opt": generate_options(
 493                             self._spec_tbs[selected["rls"]]\
 494                                 [selected["dut"]].keys()
 495                         ),
 496                         "dutver-dis": False,
 497                         "phy-val": selected["phy"],
 498                         "phy-opt": generate_options(
 499                             self._spec_tbs[selected["rls"]][selected["dut"]]\
 500                                 [selected["dutver"]].keys()
 501                         ),
 502                         "phy-dis": False,
 503                         "area-val": selected["area"],
 504                         "area-opt": [
 505                             {"label": label(v), "value": v} for v in sorted(
 506                                 self._spec_tbs[selected["rls"]]\
 507                                     [selected["dut"]][selected["dutver"]]\
 508                                         [selected["phy"]]
 509                             )
 510                         ],
 511                         "area-dis": False
 512                     })
 513                     on_draw = True
 514             elif trigger.type == "ctrl-dd":
 515                 if trigger.idx == "rls":
 516                     try:
 517                         options = generate_options(
 518                             self._spec_tbs[trigger.value].keys()
 519                         )
 520                         disabled = False
 521                     except KeyError:
 522                         options = list()
 523                         disabled = True
 524                     ctrl_panel.set({
 525                         "rls-val": trigger.value,
 526                         "dut-val": str(),
 527                         "dut-opt": options,
 528                         "dut-dis": disabled,
 529                         "dutver-val": str(),
 530                         "dutver-opt": list(),
 531                         "dutver-dis": True,
 532                         "phy-val": str(),
 533                         "phy-opt": list(),
 534                         "phy-dis": True,
 535                         "area-val": str(),
 536                         "area-opt": list(),
 537                         "area-dis": True
 538                     })
 539                 elif trigger.idx == "dut":
 540                     try:
 541                         rls = ctrl_panel.get("rls-val")
 542                         dut = self._spec_tbs[rls][trigger.value]
 543                         options = generate_options(dut.keys())
 544                         disabled = False
 545                     except KeyError:
 546                         options = list()
 547                         disabled = True
 548                     ctrl_panel.set({
 549                         "dut-val": trigger.value,
 550                         "dutver-val": str(),
 551                         "dutver-opt": options,
 552                         "dutver-dis": disabled,
 553                         "phy-val": str(),
 554                         "phy-opt": list(),
 555                         "phy-dis": True,
 556                         "area-val": str(),
 557                         "area-opt": list(),
 558                         "area-dis": True
 559                     })
 560                 elif trigger.idx == "dutver":
 561                     try:
 562                         rls = ctrl_panel.get("rls-val")
 563                         dut = ctrl_panel.get("dut-val")
 564                         dutver = self._spec_tbs[rls][dut][trigger.value]
 565                         options = generate_options(dutver.keys())
 566                         disabled = False
 567                     except KeyError:
 568                         options = list()
 569                         disabled = True
 570                     ctrl_panel.set({
 571                         "dutver-val": trigger.value,
 572                         "phy-val": str(),
 573                         "phy-opt": options,
 574                         "phy-dis": disabled,
 575                         "area-val": str(),
 576                         "area-opt": list(),
 577                         "area-dis": True
 578                     })
 579                 elif trigger.idx == "phy":
 580                     try:
 581                         rls = ctrl_panel.get("rls-val")
 582                         dut = ctrl_panel.get("dut-val")
 583                         dutver = ctrl_panel.get("dutver-val")
 584                         phy = self._spec_tbs[rls][dut][dutver][trigger.value]
 585                         options = [
 586                             {"label": label(v), "value": v} for v in sorted(phy)
 587                         ]
 588                         disabled = False
 589                     except KeyError:
 590                         options = list()
 591                         disabled = True
 592                     ctrl_panel.set({
 593                         "phy-val": trigger.value,
 594                         "area-val": str(),
 595                         "area-opt": options,
 596                         "area-dis": disabled
 597                     })
 598                 elif trigger.idx == "area":
 599                     ctrl_panel.set({"area-val": trigger.value})
 600                     selected = {
 601                         "rls": ctrl_panel.get("rls-val"),
 602                         "dut": ctrl_panel.get("dut-val"),
 603                         "dutver": ctrl_panel.get("dutver-val"),
 604                         "phy": ctrl_panel.get("phy-val"),
 605                         "area": ctrl_panel.get("area-val"),
 606                     }
 607                     on_draw = True
 608
 609             if on_draw:
 610                 if selected:
 611                     plotting_area = self._get_plotting_area(
 612                         selected,
 613                         gen_new_url(parsed_url, {"selection": selected})
 614                     )
 615                 else:
 616                     plotting_area = C.PLACEHOLDER
 617                     selected = dict()
 618
 619             ret_val = [
 620                 ctrl_panel.panel,
 621                 selected,
 622                 plotting_area,
 623             ]
 624             ret_val.extend(ctrl_panel.values)
 625             return ret_val
 626
 627         @app.callback(
 628             Output("plot-mod-url", "is_open"),
 629             [Input("plot-btn-url", "n_clicks")],
 630             [State("plot-mod-url", "is_open")],
 631         )
 632         def toggle_plot_mod_url(n, is_open):
 633             """Toggle the modal window with url.
 634             """
 635             if n:
 636                 return not is_open
 637             return is_open
 638
 639         @app.callback(
 640             Output("download-iterative-data", "data"),
 641             State("store-selected-tests", "data"),
 642             Input("plot-btn-download", "n_clicks"),
 643             prevent_initial_call=True
 644         )
 645         def _download_coverage_data(selection, _):
 646             """Download the data
 647
 648             :param selection: List of tests selected by user stored in the
 649                 browser.
 650             :type selection: dict
 651             :returns: dict of data frame content (base64 encoded) and meta data
 652                 used by the Download component.
 653             :rtype: dict
 654             """
 655
 656             if not selection:
 657                 raise PreventUpdate
 658
 659             df = select_coverage_data(self._data, selection, csv=True)
 660
 661             return dcc.send_data_frame(df.to_csv, C.COVERAGE_DOWNLOAD_FILE_NAME)