resources/tools/dash/app/pal/news/layout.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import logging
  18 import pandas as pd
  19 import dash_bootstrap_components as dbc
  20
  21 from flask import Flask
  22 from dash import dcc
  23 from dash import html
  24 from dash import callback_context
  25 from dash import Input, Output, State
  26 from yaml import load, FullLoader, YAMLError
  27 from copy import deepcopy
  28
  29 from ..data.data import Data
  30 from .tables import table_failed
  31
  32
  33 class Layout:
  34     """The layout of the dash app and the callbacks.
  35     """
  36
  37     # The default job displayed when the page is loaded first time.
  38     DEFAULT_JOB = "csit-vpp-perf-mrr-daily-master-2n-icx"
  39
  40     def __init__(self, app: Flask, html_layout_file: str, data_spec_file: str,
  41         tooltip_file: str) -> None:
  42         """Initialization:
  43         - save the input parameters,
  44         - read and pre-process the data,
  45         - prepare data fro the control panel,
  46         - read HTML layout file,
  47         - read tooltips from the tooltip file.
  48
  49         :param app: Flask application running the dash application.
  50         :param html_layout_file: Path and name of the file specifying the HTML
  51             layout of the dash application.
  52         :param data_spec_file: Path and name of the file specifying the data to
  53             be read from parquets for this application.
  54         :param tooltip_file: Path and name of the yaml file specifying the
  55             tooltips.
  56         :type app: Flask
  57         :type html_layout_file: str
  58         :type data_spec_file: str
  59         :type tooltip_file: str
  60         """
  61
  62         # Inputs
  63         self._app = app
  64         self._html_layout_file = html_layout_file
  65         self._data_spec_file = data_spec_file
  66         self._tooltip_file = tooltip_file
  67
  68         # Read the data:
  69         data_stats, data_mrr, data_ndrpdr = Data(
  70             data_spec_file=self._data_spec_file,
  71             debug=True
  72         ).read_stats(days=10)  # To be sure
  73
  74         df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
  75
  76         # Prepare information for the control panel:
  77         jobs = sorted(list(df_tst_info["job"].unique()))
  78         job_info = {
  79             "job": list(),
  80             "dut": list(),
  81             "ttype": list(),
  82             "cadence": list(),
  83             "tbed": list()
  84         }
  85         for job in jobs:
  86             lst_job = job.split("-")
  87             job_info["job"].append(job)
  88             job_info["dut"].append(lst_job[1])
  89             job_info["ttype"].append(lst_job[3])
  90             job_info["cadence"].append(lst_job[4])
  91             job_info["tbed"].append("-".join(lst_job[-2:]))
  92         self.df_job_info = pd.DataFrame.from_dict(job_info)
  93
  94         self._default = self._set_job_params(self.DEFAULT_JOB)
  95
  96         # Pre-process the data:
  97         tst_info = {
  98             "job": list(),
  99             "build": list(),
 100             "start": list(),
 101             "dut_type": list(),
 102             "dut_version": list(),
 103             "hosts": list(),
 104             "lst_failed": list()
 105         }
 106         for job in jobs:
 107             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
 108             last_build = max(df_job["build"].unique())
 109             df_build = df_job.loc[(df_job["build"] == last_build)]
 110             tst_info["job"].append(job)
 111             tst_info["build"].append(last_build)
 112             tst_info["start"].append(data_stats.loc[
 113                 (data_stats["job"] == job) &
 114                 (data_stats["build"] == last_build)
 115             ]["start_time"].iloc[-1].strftime('%Y-%m-%d %H:%M'))
 116             tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
 117             tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
 118             tst_info["hosts"].append(df_build["hosts"].iloc[-1])
 119             failed_tests = df_build.loc[(df_build["passed"] == False)]\
 120                 ["test_id"].to_list()
 121             l_failed = list()
 122             try:
 123                 for tst in failed_tests:
 124                     lst_tst = tst.split(".")
 125                     suite = lst_tst[-2].replace("2n1l-", "").\
 126                         replace("1n1l-", "").replace("2n-", "")
 127                     l_failed.append(f"{suite.split('-')[0]}-{lst_tst[-1]}")
 128             except KeyError:
 129                 l_failed = list()
 130             tst_info["lst_failed"].append(sorted(l_failed))
 131
 132         self._data = pd.DataFrame.from_dict(tst_info)
 133
 134         # Read from files:
 135         self._html_layout = str()
 136         self._tooltips = dict()
 137
 138         try:
 139             with open(self._html_layout_file, "r") as file_read:
 140                 self._html_layout = file_read.read()
 141         except IOError as err:
 142             raise RuntimeError(
 143                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 144             )
 145
 146         try:
 147             with open(self._tooltip_file, "r") as file_read:
 148                 self._tooltips = load(file_read, Loader=FullLoader)
 149         except IOError as err:
 150             logging.warning(
 151                 f"Not possible to open the file {self._tooltip_file}\n{err}"
 152             )
 153         except YAMLError as err:
 154             logging.warning(
 155                 f"An error occurred while parsing the specification file "
 156                 f"{self._tooltip_file}\n{err}"
 157             )
 158
 159         self._default_tab_failed = table_failed(self.data, self._default["job"])
 160
 161         # Callbacks:
 162         if self._app is not None and hasattr(self, 'callbacks'):
 163             self.callbacks(self._app)
 164
 165     @property
 166     def html_layout(self) -> dict:
 167         return self._html_layout
 168
 169     @property
 170     def data(self) -> pd.DataFrame:
 171         return self._data
 172
 173     @property
 174     def default(self) -> dict:
 175         return self._default
 176
 177     def _get_duts(self) -> list:
 178         """Get the list of DUTs from the pre-processed information about jobs.
 179
 180         :returns: Alphabeticaly sorted list of DUTs.
 181         :rtype: list
 182         """
 183         return sorted(list(self.df_job_info["dut"].unique()))
 184
 185     def _get_ttypes(self, dut: str) -> list:
 186         """Get the list of test types from the pre-processed information about
 187         jobs.
 188
 189         :param dut: The DUT for which the list of test types will be populated.
 190         :type dut: str
 191         :returns: Alphabeticaly sorted list of test types.
 192         :rtype: list
 193         """
 194         return sorted(list(self.df_job_info.loc[(
 195             self.df_job_info["dut"] == dut
 196         )]["ttype"].unique()))
 197
 198     def _get_cadences(self, dut: str, ttype: str) -> list:
 199         """Get the list of cadences from the pre-processed information about
 200         jobs.
 201
 202         :param dut: The DUT for which the list of cadences will be populated.
 203         :param ttype: The test type for which the list of cadences will be
 204             populated.
 205         :type dut: str
 206         :type ttype: str
 207         :returns: Alphabeticaly sorted list of cadences.
 208         :rtype: list
 209         """
 210         return sorted(list(self.df_job_info.loc[(
 211             (self.df_job_info["dut"] == dut) &
 212             (self.df_job_info["ttype"] == ttype)
 213         )]["cadence"].unique()))
 214
 215     def _get_test_beds(self, dut: str, ttype: str, cadence: str) -> list:
 216         """Get the list of test beds from the pre-processed information about
 217         jobs.
 218
 219         :param dut: The DUT for which the list of test beds will be populated.
 220         :param ttype: The test type for which the list of test beds will be
 221             populated.
 222         :param cadence: The cadence for which the list of test beds will be
 223             populated.
 224         :type dut: str
 225         :type ttype: str
 226         :type cadence: str
 227         :returns: Alphabeticaly sorted list of test beds.
 228         :rtype: list
 229         """
 230         return sorted(list(self.df_job_info.loc[(
 231             (self.df_job_info["dut"] == dut) &
 232             (self.df_job_info["ttype"] == ttype) &
 233             (self.df_job_info["cadence"] == cadence)
 234         )]["tbed"].unique()))
 235
 236     def _get_job(self, dut, ttype, cadence, testbed):
 237         """Get the name of a job defined by dut, ttype, cadence, test bed.
 238         Input information comes from the control panel.
 239
 240         :param dut: The DUT for which the job name will be created.
 241         :param ttype: The test type for which the job name will be created.
 242         :param cadence: The cadence for which the job name will be created.
 243         :param testbed: The test bed for which the job name will be created.
 244         :type dut: str
 245         :type ttype: str
 246         :type cadence: str
 247         :type testbed: str
 248         :returns: Job name.
 249         :rtype: str
 250         """
 251         return self.df_job_info.loc[(
 252             (self.df_job_info["dut"] == dut) &
 253             (self.df_job_info["ttype"] == ttype) &
 254             (self.df_job_info["cadence"] == cadence) &
 255             (self.df_job_info["tbed"] == testbed)
 256         )]["job"].item()
 257
 258     @staticmethod
 259     def _generate_options(opts: list) -> list:
 260         """Return list of options for radio items in control panel. The items in
 261         the list are dictionaries with keys "label" and "value".
 262
 263         :params opts: List of options (str) to be used for the generated list.
 264         :type opts: list
 265         :returns: List of options (dict).
 266         :rtype: list
 267         """
 268         return [{"label": i, "value": i} for i in opts]
 269
 270     def _set_job_params(self, job: str) -> dict:
 271         """Create a dictionary with all options and values for (and from) the
 272         given job.
 273
 274         :params job: The name of job for and from which the dictionary will be
 275             created.
 276         :type job: str
 277         :returns: Dictionary with all options and values for (and from) the
 278             given job.
 279         :rtype: dict
 280         """
 281
 282         lst_job = job.split("-")
 283         return {
 284             "job": job,
 285             "dut": lst_job[1],
 286             "ttype": lst_job[3],
 287             "cadence": lst_job[4],
 288             "tbed": "-".join(lst_job[-2:]),
 289             "duts": self._generate_options(self._get_duts()),
 290             "ttypes": self._generate_options(self._get_ttypes(lst_job[1])),
 291             "cadences": self._generate_options(self._get_cadences(
 292                 lst_job[1], lst_job[3])),
 293             "tbeds": self._generate_options(self._get_test_beds(
 294                 lst_job[1], lst_job[3], lst_job[4]))
 295         }
 296
 297     def _show_tooltip(self, id: str, title: str,
 298             clipboard_id: str=None) -> list:
 299         """Generate list of elements to display a text (e.g. a title) with a
 300         tooltip and optionaly with Copy&Paste icon and the clipboard
 301         functionality enabled.
 302
 303         :param id: Tooltip ID.
 304         :param title: A text for which the tooltip will be displayed.
 305         :param clipboard_id: If defined, a Copy&Paste icon is displayed and the
 306             clipboard functionality is enabled.
 307         :type id: str
 308         :type title: str
 309         :type clipboard_id: str
 310         :returns: List of elements to display a text with a tooltip and
 311             optionaly with Copy&Paste icon.
 312         :rtype: list
 313         """
 314
 315         return [
 316             dcc.Clipboard(target_id=clipboard_id, title="Copy URL") \
 317                 if clipboard_id else str(),
 318             f"{title} ",
 319             dbc.Badge(
 320                 id=id,
 321                 children="?",
 322                 pill=True,
 323                 color="white",
 324                 text_color="info",
 325                 class_name="border ms-1",
 326             ),
 327             dbc.Tooltip(
 328                 children=self._tooltips.get(id, str()),
 329                 target=id,
 330                 placement="auto"
 331             )
 332         ]
 333
 334     def add_content(self):
 335         """Top level method which generated the web page.
 336
 337         It generates:
 338         - Store for user input data,
 339         - Navigation bar,
 340         - Main area with control panel and ploting area.
 341
 342         If no HTML layout is provided, an error message is displayed instead.
 343
 344         :returns: The HTML div with teh whole page.
 345         :rtype: html.Div
 346         """
 347
 348         if self.html_layout:
 349             return html.Div(
 350                 id="div-main",
 351                 children=[
 352                     dcc.Store(id="control-panel"),
 353                     dbc.Row(
 354                         id="row-navbar",
 355                         class_name="g-0",
 356                         children=[
 357                             self._add_navbar(),
 358                         ]
 359                     ),
 360                     dbc.Row(
 361                         id="row-main",
 362                         class_name="g-0",
 363                         children=[
 364                             self._add_ctrl_col(),
 365                             self._add_plotting_col(),
 366                         ]
 367                     )
 368                 ]
 369             )
 370         else:
 371             return html.Div(
 372                 id="div-main-error",
 373                 children=[
 374                     dbc.Alert(
 375                         [
 376                             "An Error Occured",
 377                         ],
 378                         color="danger",
 379                     ),
 380                 ]
 381             )
 382
 383     def _add_navbar(self):
 384         """Add nav element with navigation panel. It is placed on the top.
 385
 386         :returns: Navigation bar.
 387         :rtype: dbc.NavbarSimple
 388         """
 389
 390         return dbc.NavbarSimple(
 391             id="navbarsimple-main",
 392             children=[
 393                 dbc.NavItem(
 394                     dbc.NavLink(
 395                         "Continuous Performance News",
 396                         disabled=True,
 397                         external_link=True,
 398                         href="#"
 399                     )
 400                 )
 401             ],
 402             brand="Dashboard",
 403             brand_href="/",
 404             brand_external_link=True,
 405             class_name="p-2",
 406             fluid=True,
 407         )
 408
 409     def _add_ctrl_col(self) -> dbc.Col:
 410         """Add column with control panel. It is placed on the left side.
 411
 412         :returns: Column with the control panel.
 413         :rtype: dbc.col
 414         """
 415
 416         return dbc.Col(
 417             id="col-controls",
 418             children=[
 419                 self._add_ctrl_panel(),
 420             ],
 421         )
 422
 423     def _add_plotting_col(self) -> dbc.Col:
 424         """Add column with tables. It is placed on the right side.
 425
 426         :returns: Column with tables.
 427         :rtype: dbc.col
 428         """
 429
 430         return dbc.Col(
 431             id="col-plotting-area",
 432             children=[
 433                 dbc.Row(  # Failed tests
 434                     id="row-table-failed",
 435                     class_name="g-0 p-2",
 436                     children=self._default_tab_failed
 437                 )
 438             ],
 439             width=9,
 440         )
 441
 442     def _add_ctrl_panel(self) -> dbc.Row:
 443         """Add control panel.
 444
 445         :returns: Control panel.
 446         :rtype: dbc.Row
 447         """
 448         return dbc.Row(
 449             id="row-ctrl-panel",
 450             class_name="g-0",
 451             children=[
 452                 dbc.Row(
 453                     class_name="g-0 p-2",
 454                     children=[
 455                         dbc.Row(
 456                             class_name="gy-1",
 457                             children=[
 458                                 dbc.Label(
 459                                     class_name="p-0",
 460                                     children=self._show_tooltip(
 461                                         "help-dut", "Device under Test")
 462                                 ),
 463                                 dbc.Row(
 464                                     dbc.RadioItems(
 465                                         id="ri-duts",
 466                                         inline=True,
 467                                         value=self.default["dut"],
 468                                         options=self.default["duts"]
 469                                     )
 470                                 )
 471                             ]
 472                         ),
 473                         dbc.Row(
 474                             class_name="gy-1",
 475                             children=[
 476                                 dbc.Label(
 477                                     class_name="p-0",
 478                                     children=self._show_tooltip(
 479                                         "help-ttype", "Test Type"),
 480                                 ),
 481                                 dbc.RadioItems(
 482                                     id="ri-ttypes",
 483                                     inline=True,
 484                                     value=self.default["ttype"],
 485                                     options=self.default["ttypes"]
 486                                 )
 487                             ]
 488                         ),
 489                         dbc.Row(
 490                             class_name="gy-1",
 491                             children=[
 492                                 dbc.Label(
 493                                     class_name="p-0",
 494                                     children=self._show_tooltip(
 495                                         "help-cadence", "Cadence"),
 496                                 ),
 497                                 dbc.RadioItems(
 498                                     id="ri-cadences",
 499                                     inline=True,
 500                                     value=self.default["cadence"],
 501                                     options=self.default["cadences"]
 502                                 )
 503                             ]
 504                         ),
 505                         dbc.Row(
 506                             class_name="gy-1",
 507                             children=[
 508                                 dbc.Label(
 509                                     class_name="p-0",
 510                                     children=self._show_tooltip(
 511                                         "help-tbed", "Test Bed"),
 512                                 ),
 513                                 dbc.Select(
 514                                     id="dd-tbeds",
 515                                     placeholder="Select a test bed...",
 516                                     value=self.default["tbed"],
 517                                     options=self.default["tbeds"]
 518                                 )
 519                             ]
 520                         ),
 521                         dbc.Row(
 522                             class_name="gy-1",
 523                             children=[
 524                                 dbc.Alert(
 525                                     id="al-job",
 526                                     color="info",
 527                                     children=self.default["job"]
 528                                 )
 529                             ]
 530                         )
 531                     ]
 532                 ),
 533             ]
 534         )
 535
 536     class ControlPanel:
 537         """
 538         """
 539
 540         def __init__(self, panel: dict, default: dict) -> None:
 541             """
 542             """
 543
 544             self._defaults = {
 545                 "ri-ttypes-options": default["ttypes"],
 546                 "ri-cadences-options": default["cadences"],
 547                 "dd-tbeds-options": default["tbeds"],
 548                 "ri-duts-value": default["dut"],
 549                 "ri-ttypes-value": default["ttype"],
 550                 "ri-cadences-value": default["cadence"],
 551                 "dd-tbeds-value": default["tbed"],
 552                 "al-job-children": default["job"]
 553             }
 554             self._panel = deepcopy(self._defaults)
 555             if panel:
 556                 for key in self._defaults:
 557                     self._panel[key] = panel[key]
 558
 559         def set(self, kwargs: dict) -> None:
 560             for key, val in kwargs.items():
 561                 if key in self._panel:
 562                     self._panel[key] = val
 563                 else:
 564                     raise KeyError(f"The key {key} is not defined.")
 565
 566         @property
 567         def defaults(self) -> dict:
 568             return self._defaults
 569
 570         @property
 571         def panel(self) -> dict:
 572             return self._panel
 573
 574         def get(self, key: str) -> any:
 575             return self._panel[key]
 576
 577         def values(self) -> list:
 578             return list(self._panel.values())
 579
 580     def callbacks(self, app):
 581
 582         @app.callback(
 583             Output("control-panel", "data"),  # Store
 584             Output("row-table-failed", "children"),
 585             Output("ri-ttypes", "options"),
 586             Output("ri-cadences", "options"),
 587             Output("dd-tbeds", "options"),
 588             Output("ri-duts", "value"),
 589             Output("ri-ttypes", "value"),
 590             Output("ri-cadences", "value"),
 591             Output("dd-tbeds", "value"),
 592             Output("al-job", "children"),
 593             State("control-panel", "data"),  # Store
 594             Input("ri-duts", "value"),
 595             Input("ri-ttypes", "value"),
 596             Input("ri-cadences", "value"),
 597             Input("dd-tbeds", "value"),
 598         )
 599         def _update_ctrl_panel(cp_data: dict, dut:str, ttype: str, cadence:str,
 600                 tbed: str) -> tuple:
 601             """
 602             """
 603
 604             ctrl_panel = self.ControlPanel(cp_data, self.default)
 605
 606             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 607             if trigger_id == "ri-duts":
 608                 ttype_opts = self._generate_options(self._get_ttypes(dut))
 609                 ttype_val = ttype_opts[0]["value"]
 610                 cad_opts = self._generate_options(
 611                     self._get_cadences(dut, ttype_val))
 612                 cad_val = cad_opts[0]["value"]
 613                 tbed_opts = self._generate_options(
 614                     self._get_test_beds(dut, ttype_val, cad_val))
 615                 tbed_val = tbed_opts[0]["value"]
 616                 ctrl_panel.set({
 617                     "ri-duts-value": dut,
 618                     "ri-ttypes-options": ttype_opts,
 619                     "ri-ttypes-value": ttype_val,
 620                     "ri-cadences-options": cad_opts,
 621                     "ri-cadences-value": cad_val,
 622                     "dd-tbeds-options": tbed_opts,
 623                     "dd-tbeds-value": tbed_val
 624                 })
 625             elif trigger_id == "ri-ttypes":
 626                 cad_opts = self._generate_options(
 627                     self._get_cadences(ctrl_panel.get("ri-duts-value"), ttype))
 628                 cad_val = cad_opts[0]["value"]
 629                 tbed_opts = self._generate_options(
 630                     self._get_test_beds(ctrl_panel.get("ri-duts-value"),
 631                     ttype, cad_val))
 632                 tbed_val = tbed_opts[0]["value"]
 633                 ctrl_panel.set({
 634                     "ri-ttypes-value": ttype,
 635                     "ri-cadences-options": cad_opts,
 636                     "ri-cadences-value": cad_val,
 637                     "dd-tbeds-options": tbed_opts,
 638                     "dd-tbeds-value": tbed_val
 639                 })
 640             elif trigger_id == "ri-cadences":
 641                 tbed_opts = self._generate_options(
 642                     self._get_test_beds(ctrl_panel.get("ri-duts-value"),
 643                     ctrl_panel.get("ri-ttypes-value"), cadence))
 644                 tbed_val = tbed_opts[0]["value"]
 645                 ctrl_panel.set({
 646                     "ri-cadences-value": cadence,
 647                     "dd-tbeds-options": tbed_opts,
 648                     "dd-tbeds-value": tbed_val
 649                 })
 650             elif trigger_id == "dd-tbeds":
 651                 ctrl_panel.set({
 652                     "dd-tbeds-value": tbed
 653                 })
 654
 655             job = self._get_job(
 656                 ctrl_panel.get("ri-duts-value"),
 657                 ctrl_panel.get("ri-ttypes-value"),
 658                 ctrl_panel.get("ri-cadences-value"),
 659                 ctrl_panel.get("dd-tbeds-value")
 660             )
 661             ctrl_panel.set({"al-job-children": job})
 662             tab_failed = table_failed(self.data, job)
 663
 664             ret_val = [
 665                 ctrl_panel.panel,
 666                 tab_failed
 667             ]
 668             ret_val.extend(ctrl_panel.values())
 669             return ret_val