resources/tools/dash/app/pal/stats/layout.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import pandas as pd
  18 import dash_bootstrap_components as dbc
  19
  20 from dash import dcc
  21 from dash import html
  22 from dash import callback_context, no_update
  23 from dash import Input, Output
  24 from dash.exceptions import PreventUpdate
  25 from yaml import load, FullLoader, YAMLError
  26 from datetime import datetime, timedelta
  27
  28 from ..data.data import Data
  29 from .graphs import graph_statistics
  30
  31
  32 class Layout:
  33     """
  34     """
  35
  36     def __init__(self, app, html_layout_file, spec_file, graph_layout_file,
  37         data_spec_file):
  38         """
  39         """
  40
  41         # Inputs
  42         self._app = app
  43         self._html_layout_file = html_layout_file
  44         self._spec_file = spec_file
  45         self._graph_layout_file = graph_layout_file
  46         self._data_spec_file = data_spec_file
  47
  48         # Read the data:
  49         data_stats, data_mrr, data_ndrpdr = Data(
  50             data_spec_file=self._data_spec_file,
  51             debug=True
  52         ).read_stats()
  53
  54         df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
  55
  56         # Pre-process the data:
  57         data_stats = data_stats[~data_stats.job.str.contains("-verify-")]
  58         data_stats = data_stats[~data_stats.job.str.contains("-coverage-")]
  59         data_stats = data_stats[~data_stats.job.str.contains("-iterative-")]
  60         data_stats = data_stats[["job", "build", "start_time", "duration"]]
  61
  62         self._jobs = sorted(list(data_stats["job"].unique()))
  63
  64         tst_info = {
  65             "job": list(),
  66             "build": list(),
  67             "dut_type": list(),
  68             "dut_version": list(),
  69             "hosts": list(),
  70             "passed": list(),
  71             "failed": list()
  72         }
  73         for job in self._jobs:
  74             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
  75             builds = df_job["build"].unique()
  76             for build in builds:
  77                 df_build = df_job.loc[(df_job["build"] == build)]
  78                 tst_info["job"].append(job)
  79                 tst_info["build"].append(build)
  80                 tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
  81                 tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
  82                 tst_info["hosts"].append(df_build["hosts"].iloc[-1])
  83                 try:
  84                     passed = df_build.value_counts(subset='passed')[True]
  85                 except KeyError:
  86                     passed = 0
  87                 try:
  88                     failed = df_build.value_counts(subset='passed')[False]
  89                 except KeyError:
  90                     failed = 0
  91                 tst_info["passed"].append(passed)
  92                 tst_info["failed"].append(failed)
  93
  94         self._data = data_stats.merge(pd.DataFrame.from_dict(tst_info))
  95
  96         # Read from files:
  97         self._html_layout = ""
  98         self._graph_layout = None
  99
 100         try:
 101             with open(self._html_layout_file, "r") as file_read:
 102                 self._html_layout = file_read.read()
 103         except IOError as err:
 104             raise RuntimeError(
 105                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 106             )
 107
 108         try:
 109             with open(self._graph_layout_file, "r") as file_read:
 110                 self._graph_layout = load(file_read, Loader=FullLoader)
 111         except IOError as err:
 112             raise RuntimeError(
 113                 f"Not possible to open the file {self._graph_layout_file}\n"
 114                 f"{err}"
 115             )
 116         except YAMLError as err:
 117             raise RuntimeError(
 118                 f"An error occurred while parsing the specification file "
 119                 f"{self._graph_layout_file}\n"
 120                 f"{err}"
 121             )
 122
 123         self._default_fig_passed, self._default_fig_duration = graph_statistics(
 124             self.data, self.jobs[0], self.layout
 125         )
 126
 127         # Callbacks:
 128         if self._app is not None and hasattr(self, 'callbacks'):
 129             self.callbacks(self._app)
 130
 131     @property
 132     def html_layout(self) -> dict:
 133         return self._html_layout
 134
 135     @property
 136     def data(self) -> pd.DataFrame:
 137         return self._data
 138
 139     @property
 140     def layout(self) -> dict:
 141         return self._graph_layout
 142
 143     @property
 144     def jobs(self) -> list:
 145         return self._jobs
 146
 147     def add_content(self):
 148         """
 149         """
 150         if self.html_layout:
 151             return html.Div(
 152                 id="div-main",
 153                 children=[
 154                     dbc.Row(
 155                         id="row-navbar",
 156                         class_name="g-0",
 157                         children=[
 158                             self._add_navbar(),
 159                         ]
 160                     ),
 161                     dcc.Loading(
 162                         dbc.Offcanvas(
 163                             class_name="w-50",
 164                             id="offcanvas-metadata",
 165                             title="Detailed Information",
 166                             placement="end",
 167                             is_open=False,
 168                             children=[
 169                                 dbc.Row(id="row-metadata")
 170                             ]
 171                         )
 172                     ),
 173                     dbc.Row(
 174                         id="row-main",
 175                         class_name="g-0",
 176                         children=[
 177                             self._add_ctrl_col(),
 178                             self._add_plotting_col(),
 179                         ]
 180                     )
 181                 ]
 182             )
 183         else:
 184             return html.Div(
 185                 id="div-main-error",
 186                 children=[
 187                     dbc.Alert(
 188                         [
 189                             "An Error Occured",
 190                         ],
 191                         color="danger",
 192                     ),
 193                 ]
 194             )
 195
 196     def _add_navbar(self):
 197         """Add nav element with navigation panel. It is placed on the top.
 198         """
 199         return dbc.NavbarSimple(
 200             id="navbarsimple-main",
 201             children=[
 202                 dbc.NavItem(
 203                     dbc.NavLink(
 204                         "Continuous Performance Statistics",
 205                         disabled=True,
 206                         external_link=True,
 207                         href="#"
 208                     )
 209                 )
 210             ],
 211             brand="Dashboard",
 212             brand_href="/",
 213             brand_external_link=True,
 214             class_name="p-2",
 215             fluid=True,
 216         )
 217
 218     def _add_ctrl_col(self) -> dbc.Col:
 219         """Add column with controls. It is placed on the left side.
 220         """
 221         return dbc.Col(
 222             id="col-controls",
 223             children=[
 224                 self._add_ctrl_panel(),
 225             ],
 226         )
 227
 228     def _add_plotting_col(self) -> dbc.Col:
 229         """Add column with plots and tables. It is placed on the right side.
 230         """
 231         return dbc.Col(
 232             id="col-plotting-area",
 233             children=[
 234                 dbc.Row(  # Passed / failed tests
 235                     id="row-graph-passed",
 236                     class_name="g-0 p-2",
 237                     children=[
 238                         dcc.Loading(children=[
 239                             dcc.Graph(
 240                                 id="graph-passed",
 241                                 figure=self._default_fig_passed
 242                             )
 243                         ])
 244                     ]
 245                 ),
 246                 dbc.Row(  # Duration
 247                     id="row-graph-duration",
 248                     class_name="g-0 p-2",
 249                     children=[
 250                         dcc.Loading(children=[
 251                             dcc.Graph(
 252                                 id="graph-duration",
 253                                 figure=self._default_fig_duration
 254                             )
 255                         ])
 256                     ]
 257                 ),
 258                 dbc.Row(  # Download
 259                     id="row-btn-download",
 260                     class_name="g-0 p-2",
 261                     children=[
 262                         dcc.Loading(children=[
 263                             dbc.Button(
 264                                 id="btn-download-data",
 265                                 children=["Download Data"],
 266                                 class_name="me-1",
 267                                 color="info"
 268                             ),
 269                             dcc.Download(id="download-data")
 270                         ])
 271                     ]
 272                 )
 273             ],
 274             width=9,
 275         )
 276
 277     def _add_ctrl_panel(self) -> dbc.Row:
 278         """
 279         """
 280         return dbc.Row(
 281             id="row-ctrl-panel",
 282             class_name="g-0",
 283             children=[
 284                 dbc.Row(
 285                     class_name="g-0 p-2",
 286                     children=[
 287                         dbc.Label("Choose the Trending Job"),
 288                         dbc.RadioItems(
 289                             id="ri_job",
 290                             value=self.jobs[0],
 291                             options=[
 292                                 {"label": i, "value": i} for i in self.jobs
 293                             ]
 294                         )
 295                     ]
 296                 ),
 297                 dbc.Row(
 298                     class_name="g-0 p-2",
 299                     children=[
 300                         dbc.Label("Choose the Time Period"),
 301                         dcc.DatePickerRange(
 302                             id="dpr-period",
 303                             className="d-flex justify-content-center",
 304                             min_date_allowed=\
 305                                 datetime.utcnow()-timedelta(days=180),
 306                             max_date_allowed=datetime.utcnow(),
 307                             initial_visible_month=datetime.utcnow(),
 308                             start_date=datetime.utcnow() - timedelta(days=180),
 309                             end_date=datetime.utcnow(),
 310                             display_format="D MMMM YY"
 311                         )
 312                     ]
 313                 )
 314             ]
 315         )
 316
 317     def callbacks(self, app):
 318
 319         @app.callback(
 320             Output("graph-passed", "figure"),
 321             Output("graph-duration", "figure"),
 322             Input("ri_job", "value"),
 323             Input("dpr-period", "start_date"),
 324             Input("dpr-period", "end_date"),
 325             prevent_initial_call=True
 326         )
 327         def _update_ctrl_panel(job:str, d_start: str, d_end: str) -> tuple:
 328             """
 329             """
 330
 331             d_start = datetime(int(d_start[0:4]), int(d_start[5:7]),
 332                 int(d_start[8:10]))
 333             d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10]))
 334
 335             fig_passed, fig_duration = graph_statistics(
 336                 self.data, job, self.layout, d_start, d_end
 337             )
 338
 339             return fig_passed, fig_duration
 340
 341         @app.callback(
 342             Output("download-data", "data"),
 343             Input("btn-download-data", "n_clicks"),
 344             prevent_initial_call=True
 345         )
 346         def _download_data(n_clicks):
 347             """
 348             """
 349             if not n_clicks:
 350                 raise PreventUpdate
 351
 352             return dcc.send_data_frame(self.data.to_csv, "statistics.csv")
 353
 354         @app.callback(
 355             Output("row-metadata", "children"),
 356             Output("offcanvas-metadata", "is_open"),
 357             Input("graph-passed", "clickData"),
 358             Input("graph-duration", "clickData"),
 359             prevent_initial_call=True
 360         )
 361         def _show_metadata_from_graphs(
 362                 passed_data: dict, duration_data: dict) -> tuple:
 363             """
 364             """
 365
 366             if not (passed_data or duration_data):
 367                 raise PreventUpdate
 368
 369             metadata = no_update
 370             open_canvas = False
 371             title = "Job Statistics"
 372             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 373             if trigger_id == "graph-passed":
 374                 graph_data = passed_data["points"][0].get("hovertext", "")
 375             elif trigger_id == "graph-duration":
 376                 graph_data = duration_data["points"][0].get("text", "")
 377             if graph_data:
 378                 metadata = [
 379                     dbc.Card(
 380                         class_name="gy-2 p-0",
 381                         children=[
 382                             dbc.CardHeader(children=[
 383                                 dcc.Clipboard(
 384                                     target_id="metadata",
 385                                     title="Copy",
 386                                     style={"display": "inline-block"}
 387                                 ),
 388                                 title
 389                             ]),
 390                             dbc.CardBody(
 391                                 id="metadata",
 392                                 class_name="p-0",
 393                                 children=[dbc.ListGroup(
 394                                     children=[
 395                                         dbc.ListGroupItem(
 396                                             [
 397                                                 dbc.Badge(
 398                                                     x.split(":")[0]
 399                                                 ),
 400                                                 x.split(": ")[1]
 401                                             ]
 402                                         ) for x in graph_data.split("<br>")
 403                                     ],
 404                                     flush=True),
 405                                 ]
 406                             )
 407                         ]
 408                     )
 409                 ]
 410                 open_canvas = True
 411
 412             return metadata, open_canvas