resources/tools/dash/app/pal/news/layout.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import logging
  18 import pandas as pd
  19 import dash_bootstrap_components as dbc
  20
  21 from flask import Flask
  22 from dash import dcc
  23 from dash import html
  24 from dash import callback_context
  25 from dash import Input, Output, State
  26 from yaml import load, FullLoader, YAMLError
  27 from copy import deepcopy
  28
  29 from ..data.data import Data
  30 from .tables import table_failed
  31
  32
  33 class Layout:
  34     """
  35     """
  36
  37     DEFAULT_JOB = "csit-vpp-perf-mrr-daily-master-2n-icx"
  38
  39     URL_STYLE = {
  40         "background-color": "#d2ebf5",
  41         "border-color": "#bce1f1",
  42         "color": "#135d7c"
  43     }
  44
  45     def __init__(self, app: Flask, html_layout_file: str,
  46         data_spec_file: str, tooltip_file: str) -> None:
  47         """
  48         """
  49
  50         # Inputs
  51         self._app = app
  52         self._html_layout_file = html_layout_file
  53         self._data_spec_file = data_spec_file
  54         self._tooltip_file = tooltip_file
  55
  56         # Read the data:
  57         data_stats, data_mrr, data_ndrpdr = Data(
  58             data_spec_file=self._data_spec_file,
  59             debug=True
  60         ).read_stats(days=10)  # To be sure
  61
  62         df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
  63
  64         jobs = sorted(list(df_tst_info["job"].unique()))
  65         job_info = {
  66             "job": list(),
  67             "dut": list(),
  68             "ttype": list(),
  69             "cadence": list(),
  70             "tbed": list()
  71         }
  72         for job in jobs:
  73             lst_job = job.split("-")
  74             job_info["job"].append(job)
  75             job_info["dut"].append(lst_job[1])
  76             job_info["ttype"].append(lst_job[3])
  77             job_info["cadence"].append(lst_job[4])
  78             job_info["tbed"].append("-".join(lst_job[-2:]))
  79         self.df_job_info = pd.DataFrame.from_dict(job_info)
  80
  81         self._default = self._set_job_params(self.DEFAULT_JOB)
  82
  83         tst_info = {
  84             "job": list(),
  85             "build": list(),
  86             "start": list(),
  87             "dut_type": list(),
  88             "dut_version": list(),
  89             "hosts": list(),
  90             "lst_failed": list()
  91         }
  92         for job in jobs:
  93             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
  94             last_build = max(df_job["build"].unique())
  95             df_build = df_job.loc[(df_job["build"] == last_build)]
  96             tst_info["job"].append(job)
  97             tst_info["build"].append(last_build)
  98             tst_info["start"].append(data_stats.loc[
  99                 (data_stats["job"] == job) &
 100                 (data_stats["build"] == last_build)
 101             ]["start_time"].iloc[-1].strftime('%Y-%m-%d %H:%M'))
 102             tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
 103             tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
 104             tst_info["hosts"].append(df_build["hosts"].iloc[-1])
 105             failed_tests = df_build.loc[(df_build["passed"] == False)]\
 106                 ["test_id"].to_list()
 107             l_failed = list()
 108             try:
 109                 for tst in failed_tests:
 110                     lst_tst = tst.split(".")
 111                     suite = lst_tst[-2].replace("2n1l-", "").\
 112                         replace("1n1l-", "").replace("2n-", "")
 113                     l_failed.append(f"{suite.split('-')[0]}-{lst_tst[-1]}")
 114             except KeyError:
 115                 l_failed = list()
 116             tst_info["lst_failed"].append(sorted(l_failed))
 117
 118         self._data = pd.DataFrame.from_dict(tst_info)
 119
 120         # Read from files:
 121         self._html_layout = ""
 122         self._tooltips = dict()
 123
 124         try:
 125             with open(self._html_layout_file, "r") as file_read:
 126                 self._html_layout = file_read.read()
 127         except IOError as err:
 128             raise RuntimeError(
 129                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 130             )
 131
 132         try:
 133             with open(self._tooltip_file, "r") as file_read:
 134                 self._tooltips = load(file_read, Loader=FullLoader)
 135         except IOError as err:
 136             logging.warning(
 137                 f"Not possible to open the file {self._tooltip_file}\n{err}"
 138             )
 139         except YAMLError as err:
 140             logging.warning(
 141                 f"An error occurred while parsing the specification file "
 142                 f"{self._tooltip_file}\n{err}"
 143             )
 144
 145         self._default_tab_failed = table_failed(self.data, self._default["job"])
 146
 147         # Callbacks:
 148         if self._app is not None and hasattr(self, 'callbacks'):
 149             self.callbacks(self._app)
 150
 151     @property
 152     def html_layout(self) -> dict:
 153         return self._html_layout
 154
 155     @property
 156     def data(self) -> pd.DataFrame:
 157         return self._data
 158
 159     @property
 160     def default(self) -> any:
 161         return self._default
 162
 163     def _get_duts(self) -> list:
 164         """
 165         """
 166         return sorted(list(self.df_job_info["dut"].unique()))
 167
 168     def _get_ttypes(self, dut: str) -> list:
 169         """
 170         """
 171         return sorted(list(self.df_job_info.loc[(
 172             self.df_job_info["dut"] == dut
 173         )]["ttype"].unique()))
 174
 175     def _get_cadences(self, dut: str, ttype: str) -> list:
 176         """
 177         """
 178         return sorted(list(self.df_job_info.loc[(
 179             (self.df_job_info["dut"] == dut) &
 180             (self.df_job_info["ttype"] == ttype)
 181         )]["cadence"].unique()))
 182
 183     def _get_test_beds(self, dut: str, ttype: str, cadence: str) -> list:
 184         """
 185         """
 186         return sorted(list(self.df_job_info.loc[(
 187             (self.df_job_info["dut"] == dut) &
 188             (self.df_job_info["ttype"] == ttype) &
 189             (self.df_job_info["cadence"] == cadence)
 190         )]["tbed"].unique()))
 191
 192     def _get_job(self, dut, ttype, cadence, testbed):
 193         """Get the name of a job defined by dut, ttype, cadence, testbed.
 194
 195         Input information comes from control panel.
 196         """
 197         return self.df_job_info.loc[(
 198             (self.df_job_info["dut"] == dut) &
 199             (self.df_job_info["ttype"] == ttype) &
 200             (self.df_job_info["cadence"] == cadence) &
 201             (self.df_job_info["tbed"] == testbed)
 202         )]["job"].item()
 203
 204     def _set_job_params(self, job: str) -> dict:
 205         """
 206         """
 207         lst_job = job.split("-")
 208         return {
 209             "job": job,
 210             "dut": lst_job[1],
 211             "ttype": lst_job[3],
 212             "cadence": lst_job[4],
 213             "tbed": "-".join(lst_job[-2:]),
 214             "duts": self._generate_options(self._get_duts()),
 215             "ttypes": self._generate_options(self._get_ttypes(lst_job[1])),
 216             "cadences": self._generate_options(self._get_cadences(
 217                 lst_job[1], lst_job[3])),
 218             "tbeds": self._generate_options(self._get_test_beds(
 219                 lst_job[1], lst_job[3], lst_job[4]))
 220         }
 221
 222     def _show_tooltip(self, id: str, title: str,
 223             clipboard_id: str=None) -> list:
 224         """
 225         """
 226         return [
 227             dcc.Clipboard(target_id=clipboard_id, title="Copy URL") \
 228                 if clipboard_id else str(),
 229             f"{title} ",
 230             dbc.Badge(
 231                 id=id,
 232                 children="?",
 233                 pill=True,
 234                 color="white",
 235                 text_color="info",
 236                 class_name="border ms-1",
 237             ),
 238             dbc.Tooltip(
 239                 children=self._tooltips.get(id, str()),
 240                 target=id,
 241                 placement="auto"
 242             )
 243         ]
 244
 245     def add_content(self):
 246         """
 247         """
 248         if self.html_layout:
 249             return html.Div(
 250                 id="div-main",
 251                 children=[
 252                     dcc.Store(id="control-panel"),
 253                     dbc.Row(
 254                         id="row-navbar",
 255                         class_name="g-0",
 256                         children=[
 257                             self._add_navbar(),
 258                         ]
 259                     ),
 260                     dbc.Row(
 261                         id="row-main",
 262                         class_name="g-0",
 263                         children=[
 264                             self._add_ctrl_col(),
 265                             self._add_plotting_col(),
 266                         ]
 267                     )
 268                 ]
 269             )
 270         else:
 271             return html.Div(
 272                 id="div-main-error",
 273                 children=[
 274                     dbc.Alert(
 275                         [
 276                             "An Error Occured",
 277                         ],
 278                         color="danger",
 279                     ),
 280                 ]
 281             )
 282
 283     def _add_navbar(self):
 284         """Add nav element with navigation panel. It is placed on the top.
 285         """
 286         return dbc.NavbarSimple(
 287             id="navbarsimple-main",
 288             children=[
 289                 dbc.NavItem(
 290                     dbc.NavLink(
 291                         "Continuous Performance News",
 292                         disabled=True,
 293                         external_link=True,
 294                         href="#"
 295                     )
 296                 )
 297             ],
 298             brand="Dashboard",
 299             brand_href="/",
 300             brand_external_link=True,
 301             class_name="p-2",
 302             fluid=True,
 303         )
 304
 305     def _add_ctrl_col(self) -> dbc.Col:
 306         """Add column with controls. It is placed on the left side.
 307         """
 308         return dbc.Col(
 309             id="col-controls",
 310             children=[
 311                 self._add_ctrl_panel(),
 312             ],
 313         )
 314
 315     def _add_plotting_col(self) -> dbc.Col:
 316         """Add column with plots and tables. It is placed on the right side.
 317         """
 318         return dbc.Col(
 319             id="col-plotting-area",
 320             children=[
 321                 dbc.Row(  # Failed tests
 322                     id="row-table-failed",
 323                     class_name="g-0 p-2",
 324                     children=self._default_tab_failed
 325                 )
 326             ],
 327             width=9,
 328         )
 329
 330     def _add_ctrl_panel(self) -> dbc.Row:
 331         """
 332         """
 333         return dbc.Row(
 334             id="row-ctrl-panel",
 335             class_name="g-0",
 336             children=[
 337                 dbc.Row(
 338                     class_name="g-0 p-2",
 339                     children=[
 340                         dbc.Row(
 341                             class_name="gy-1",
 342                             children=[
 343                                 dbc.Label(
 344                                     class_name="p-0",
 345                                     children=self._show_tooltip(
 346                                         "help-dut", "Device under Test")
 347                                 ),
 348                                 dbc.Row(
 349                                     dbc.RadioItems(
 350                                         id="ri-duts",
 351                                         inline=True,
 352                                         value=self.default["dut"],
 353                                         options=self.default["duts"]
 354                                     )
 355                                 )
 356                             ]
 357                         ),
 358                         dbc.Row(
 359                             class_name="gy-1",
 360                             children=[
 361                                 dbc.Label(
 362                                     class_name="p-0",
 363                                     children=self._show_tooltip(
 364                                         "help-ttype", "Test Type"),
 365                                 ),
 366                                 dbc.RadioItems(
 367                                     id="ri-ttypes",
 368                                     inline=True,
 369                                     value=self.default["ttype"],
 370                                     options=self.default["ttypes"]
 371                                 )
 372                             ]
 373                         ),
 374                         dbc.Row(
 375                             class_name="gy-1",
 376                             children=[
 377                                 dbc.Label(
 378                                     class_name="p-0",
 379                                     children=self._show_tooltip(
 380                                         "help-cadence", "Cadence"),
 381                                 ),
 382                                 dbc.RadioItems(
 383                                     id="ri-cadences",
 384                                     inline=True,
 385                                     value=self.default["cadence"],
 386                                     options=self.default["cadences"]
 387                                 )
 388                             ]
 389                         ),
 390                         dbc.Row(
 391                             class_name="gy-1",
 392                             children=[
 393                                 dbc.Label(
 394                                     class_name="p-0",
 395                                     children=self._show_tooltip(
 396                                         "help-tbed", "Test Bed"),
 397                                 ),
 398                                 dbc.Select(
 399                                     id="dd-tbeds",
 400                                     placeholder="Select a test bed...",
 401                                     value=self.default["tbed"],
 402                                     options=self.default["tbeds"]
 403                                 )
 404                             ]
 405                         ),
 406                         dbc.Row(
 407                             class_name="gy-1",
 408                             children=[
 409                                 dbc.Alert(
 410                                     id="al-job",
 411                                     color="info",
 412                                     children=self.default["job"]
 413                                 )
 414                             ]
 415                         )
 416                     ]
 417                 ),
 418             ]
 419         )
 420
 421     class ControlPanel:
 422         def __init__(self, panel: dict, default: dict) -> None:
 423             self._defaults = {
 424                 "ri-ttypes-options": default["ttypes"],
 425                 "ri-cadences-options": default["cadences"],
 426                 "dd-tbeds-options": default["tbeds"],
 427                 "ri-duts-value": default["dut"],
 428                 "ri-ttypes-value": default["ttype"],
 429                 "ri-cadences-value": default["cadence"],
 430                 "dd-tbeds-value": default["tbed"],
 431                 "al-job-children": default["job"]
 432             }
 433             self._panel = deepcopy(self._defaults)
 434             if panel:
 435                 for key in self._defaults:
 436                     self._panel[key] = panel[key]
 437
 438         def set(self, kwargs: dict) -> None:
 439             for key, val in kwargs.items():
 440                 if key in self._panel:
 441                     self._panel[key] = val
 442                 else:
 443                     raise KeyError(f"The key {key} is not defined.")
 444
 445         @property
 446         def defaults(self) -> dict:
 447             return self._defaults
 448
 449         @property
 450         def panel(self) -> dict:
 451             return self._panel
 452
 453         def get(self, key: str) -> any:
 454             return self._panel[key]
 455
 456         def values(self) -> list:
 457             return list(self._panel.values())
 458
 459     @staticmethod
 460     def _generate_options(opts: list) -> list:
 461         return [{"label": i, "value": i} for i in opts]
 462
 463     def callbacks(self, app):
 464
 465         @app.callback(
 466             Output("control-panel", "data"),  # Store
 467             Output("row-table-failed", "children"),
 468             Output("ri-ttypes", "options"),
 469             Output("ri-cadences", "options"),
 470             Output("dd-tbeds", "options"),
 471             Output("ri-duts", "value"),
 472             Output("ri-ttypes", "value"),
 473             Output("ri-cadences", "value"),
 474             Output("dd-tbeds", "value"),
 475             Output("al-job", "children"),
 476             State("control-panel", "data"),  # Store
 477             Input("ri-duts", "value"),
 478             Input("ri-ttypes", "value"),
 479             Input("ri-cadences", "value"),
 480             Input("dd-tbeds", "value"),
 481         )
 482         def _update_ctrl_panel(cp_data: dict, dut:str, ttype: str, cadence:str,
 483                 tbed: str) -> tuple:
 484             """
 485             """
 486
 487             ctrl_panel = self.ControlPanel(cp_data, self.default)
 488
 489             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 490             if trigger_id == "ri-duts":
 491                 ttype_opts = self._generate_options(self._get_ttypes(dut))
 492                 ttype_val = ttype_opts[0]["value"]
 493                 cad_opts = self._generate_options(
 494                     self._get_cadences(dut, ttype_val))
 495                 cad_val = cad_opts[0]["value"]
 496                 tbed_opts = self._generate_options(
 497                     self._get_test_beds(dut, ttype_val, cad_val))
 498                 tbed_val = tbed_opts[0]["value"]
 499                 ctrl_panel.set({
 500                     "ri-duts-value": dut,
 501                     "ri-ttypes-options": ttype_opts,
 502                     "ri-ttypes-value": ttype_val,
 503                     "ri-cadences-options": cad_opts,
 504                     "ri-cadences-value": cad_val,
 505                     "dd-tbeds-options": tbed_opts,
 506                     "dd-tbeds-value": tbed_val
 507                 })
 508             elif trigger_id == "ri-ttypes":
 509                 cad_opts = self._generate_options(
 510                     self._get_cadences(ctrl_panel.get("ri-duts-value"), ttype))
 511                 cad_val = cad_opts[0]["value"]
 512                 tbed_opts = self._generate_options(
 513                     self._get_test_beds(ctrl_panel.get("ri-duts-value"),
 514                     ttype, cad_val))
 515                 tbed_val = tbed_opts[0]["value"]
 516                 ctrl_panel.set({
 517                     "ri-ttypes-value": ttype,
 518                     "ri-cadences-options": cad_opts,
 519                     "ri-cadences-value": cad_val,
 520                     "dd-tbeds-options": tbed_opts,
 521                     "dd-tbeds-value": tbed_val
 522                 })
 523             elif trigger_id == "ri-cadences":
 524                 tbed_opts = self._generate_options(
 525                     self._get_test_beds(ctrl_panel.get("ri-duts-value"),
 526                     ctrl_panel.get("ri-ttypes-value"), cadence))
 527                 tbed_val = tbed_opts[0]["value"]
 528                 ctrl_panel.set({
 529                     "ri-cadences-value": cadence,
 530                     "dd-tbeds-options": tbed_opts,
 531                     "dd-tbeds-value": tbed_val
 532                 })
 533             elif trigger_id == "dd-tbeds":
 534                 ctrl_panel.set({
 535                     "dd-tbeds-value": tbed
 536                 })
 537
 538             job = self._get_job(
 539                 ctrl_panel.get("ri-duts-value"),
 540                 ctrl_panel.get("ri-ttypes-value"),
 541                 ctrl_panel.get("ri-cadences-value"),
 542                 ctrl_panel.get("dd-tbeds-value")
 543             )
 544             ctrl_panel.set({"al-job-children": job})
 545             tab_failed = table_failed(self.data, job)
 546
 547             ret_val = [
 548                 ctrl_panel.panel,
 549                 tab_failed
 550             ]
 551             ret_val.extend(ctrl_panel.values())
 552             return ret_val