resources/tools/dash/app/pal/stats/layout.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import pandas as pd
  18 import dash_bootstrap_components as dbc
  19
  20 from dash import dcc
  21 from dash import html
  22 from dash import Input, Output
  23 from dash.exceptions import PreventUpdate
  24 from yaml import load, FullLoader, YAMLError
  25 from datetime import datetime, timedelta
  26
  27 from ..data.data import Data
  28 from .graphs import graph_statistics
  29
  30
  31 class Layout:
  32     """
  33     """
  34
  35     def __init__(self, app, html_layout_file, spec_file, graph_layout_file,
  36         data_spec_file):
  37         """
  38         """
  39
  40         # Inputs
  41         self._app = app
  42         self._html_layout_file = html_layout_file
  43         self._spec_file = spec_file
  44         self._graph_layout_file = graph_layout_file
  45         self._data_spec_file = data_spec_file
  46
  47         # Read the data:
  48         data_stats, data_mrr, data_ndrpdr = Data(
  49             data_spec_file=self._data_spec_file,
  50             debug=True
  51         ).read_stats(days=180)
  52
  53         df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
  54
  55         # Pre-process the data:
  56         data_stats = data_stats[~data_stats.job.str.contains("-verify-")]
  57         data_stats = data_stats[~data_stats.job.str.contains("-coverage-")]
  58         data_stats = data_stats[~data_stats.job.str.contains("-iterative-")]
  59         data_stats = data_stats[["job", "build", "start_time", "duration"]]
  60
  61         self._jobs = sorted(list(data_stats["job"].unique()))
  62
  63         tst_info = {
  64             "job": list(),
  65             "build": list(),
  66             "dut_type": list(),
  67             "dut_version": list(),
  68             "hosts": list(),
  69             "passed": list(),
  70             "failed": list()
  71         }
  72         for job in self._jobs:
  73             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
  74             builds = df_job["build"].unique()
  75             for build in builds:
  76                 df_build = df_job.loc[(df_job["build"] == build)]
  77                 tst_info["job"].append(job)
  78                 tst_info["build"].append(build)
  79                 tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
  80                 tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
  81                 tst_info["hosts"].append(df_build["hosts"].iloc[-1])
  82                 try:
  83                     passed = df_build.value_counts(subset='passed')[True]
  84                 except KeyError:
  85                     passed = 0
  86                 try:
  87                     failed = df_build.value_counts(subset='passed')[False]
  88                 except KeyError:
  89                     failed = 0
  90                 tst_info["passed"].append(passed)
  91                 tst_info["failed"].append(failed)
  92
  93         self._data = data_stats.merge(pd.DataFrame.from_dict(tst_info))
  94
  95         # Read from files:
  96         self._html_layout = ""
  97         self._graph_layout = None
  98
  99         try:
 100             with open(self._html_layout_file, "r") as file_read:
 101                 self._html_layout = file_read.read()
 102         except IOError as err:
 103             raise RuntimeError(
 104                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 105             )
 106
 107         try:
 108             with open(self._graph_layout_file, "r") as file_read:
 109                 self._graph_layout = load(file_read, Loader=FullLoader)
 110         except IOError as err:
 111             raise RuntimeError(
 112                 f"Not possible to open the file {self._graph_layout_file}\n"
 113                 f"{err}"
 114             )
 115         except YAMLError as err:
 116             raise RuntimeError(
 117                 f"An error occurred while parsing the specification file "
 118                 f"{self._graph_layout_file}\n"
 119                 f"{err}"
 120             )
 121
 122         self._default_fig_passed, self._default_fig_duration = graph_statistics(
 123             self.data, self.jobs[0], self.layout
 124         )
 125
 126         # Callbacks:
 127         if self._app is not None and hasattr(self, 'callbacks'):
 128             self.callbacks(self._app)
 129
 130     @property
 131     def html_layout(self) -> dict:
 132         return self._html_layout
 133
 134     @property
 135     def data(self) -> pd.DataFrame:
 136         return self._data
 137
 138     @property
 139     def layout(self) -> dict:
 140         return self._graph_layout
 141
 142     @property
 143     def jobs(self) -> list:
 144         return self._jobs
 145
 146     def add_content(self):
 147         """
 148         """
 149         if self.html_layout:
 150             return html.Div(
 151                 id="div-main",
 152                 children=[
 153                     dbc.Row(
 154                         id="row-navbar",
 155                         class_name="g-0",
 156                         children=[
 157                             self._add_navbar(),
 158                         ]
 159                     ),
 160                     dbc.Row(
 161                         id="row-main",
 162                         class_name="g-0",
 163                         children=[
 164                             dcc.Store(
 165                                 id="selected-tests"
 166                             ),
 167                             dcc.Store(
 168                                 id="control-panel"
 169                             ),
 170                             self._add_ctrl_col(),
 171                             self._add_plotting_col(),
 172                         ]
 173                     )
 174                 ]
 175             )
 176         else:
 177             return html.Div(
 178                 id="div-main-error",
 179                 children=[
 180                     dbc.Alert(
 181                         [
 182                             "An Error Occured",
 183                         ],
 184                         color="danger",
 185                     ),
 186                 ]
 187             )
 188
 189     def _add_navbar(self):
 190         """Add nav element with navigation panel. It is placed on the top.
 191         """
 192         return dbc.NavbarSimple(
 193             id="navbarsimple-main",
 194             children=[
 195                 dbc.NavItem(
 196                     dbc.NavLink(
 197                         "Continuous Performance Statistics",
 198                         disabled=True,
 199                         external_link=True,
 200                         href="#"
 201                     )
 202                 )
 203             ],
 204             brand="Dashboard",
 205             brand_href="/",
 206             brand_external_link=True,
 207             class_name="p-2",
 208             fluid=True,
 209         )
 210
 211     def _add_ctrl_col(self) -> dbc.Col:
 212         """Add column with controls. It is placed on the left side.
 213         """
 214         return dbc.Col(
 215             id="col-controls",
 216             children=[
 217                 self._add_ctrl_panel(),
 218             ],
 219         )
 220
 221     def _add_plotting_col(self) -> dbc.Col:
 222         """Add column with plots and tables. It is placed on the right side.
 223         """
 224         return dbc.Col(
 225             id="col-plotting-area",
 226             children=[
 227                 dbc.Row(  # Passed / failed tests
 228                     id="row-graph-passed",
 229                     class_name="g-0 p-2",
 230                     children=[
 231                         dcc.Loading(children=[
 232                             dcc.Graph(
 233                                 id="graph-passed",
 234                                 figure=self._default_fig_passed
 235                             )
 236                         ])
 237                     ]
 238                 ),
 239                 dbc.Row(  # Duration
 240                     id="row-graph-duration",
 241                     class_name="g-0 p-2",
 242                     children=[
 243                         dcc.Loading(children=[
 244                             dcc.Graph(
 245                                 id="graph-duration",
 246                                 figure=self._default_fig_duration
 247                             )
 248                         ])
 249                     ]
 250                 ),
 251                 dbc.Row(  # Download
 252                     id="row-btn-download",
 253                     class_name="g-0 p-2",
 254                     children=[
 255                         dcc.Loading(children=[
 256                             dbc.Button(
 257                                 id="btn-download-data",
 258                                 children=["Download Data"]
 259                             ),
 260                             dcc.Download(id="download-data")
 261                         ])
 262                     ]
 263                 )
 264             ],
 265             width=9,
 266         )
 267
 268     def _add_ctrl_panel(self) -> dbc.Row:
 269         """
 270         """
 271         return dbc.Row(
 272             id="row-ctrl-panel",
 273             class_name="g-0 p-2",
 274             children=[
 275                 dbc.Label("Choose the Trending Job"),
 276                 dbc.RadioItems(
 277                     id="ri_job",
 278                     value=self.jobs[0],
 279                     options=[{"label": i, "value": i} for i in self.jobs]
 280                 ),
 281                 dbc.Label("Choose the Time Period"),
 282                 dcc.DatePickerRange(
 283                     id="dpr-period",
 284                     className="d-flex justify-content-center",
 285                     min_date_allowed=\
 286                         datetime.utcnow()-timedelta(days=180),
 287                     max_date_allowed=datetime.utcnow(),
 288                     initial_visible_month=datetime.utcnow(),
 289                     start_date=datetime.utcnow() - timedelta(days=180),
 290                     end_date=datetime.utcnow(),
 291                     display_format="D MMMM YY"
 292                 )
 293             ]
 294         )
 295
 296     def callbacks(self, app):
 297
 298         @app.callback(
 299             Output("graph-passed", "figure"),
 300             Output("graph-duration", "figure"),
 301             Input("ri_job", "value"),
 302             Input("dpr-period", "start_date"),
 303             Input("dpr-period", "end_date"),
 304             prevent_initial_call=True
 305         )
 306         def _update_ctrl_panel(job:str, d_start: str, d_end: str) -> tuple:
 307             """
 308             """
 309
 310             d_start = datetime(int(d_start[0:4]), int(d_start[5:7]),
 311                 int(d_start[8:10]))
 312             d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10]))
 313
 314             fig_passed, fig_duration = graph_statistics(
 315                 self.data, job, self.layout, d_start, d_end
 316             )
 317
 318             return fig_passed, fig_duration
 319
 320         @app.callback(
 321             Output("download-data", "data"),
 322             Input("btn-download-data", "n_clicks"),
 323             prevent_initial_call=True
 324         )
 325         def _download_data(n_clicks):
 326             """
 327             """
 328             if not n_clicks:
 329                 raise PreventUpdate
 330
 331             return dcc.send_data_frame(self.data.to_csv, "statistics.csv")