csit.infra.dash/app/pal/news/layout.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import logging
  18 import pandas as pd
  19 import dash_bootstrap_components as dbc
  20
  21 from flask import Flask
  22 from dash import dcc
  23 from dash import html
  24 from dash import callback_context
  25 from dash import Input, Output
  26 from yaml import load, FullLoader, YAMLError
  27
  28 from ..data.data import Data
  29 from ..utils.constants import Constants as C
  30 from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url
  31 from ..utils.url_processing import url_decode
  32 from ..data.data import Data
  33 from .tables import table_summary
  34
  35
  36 class Layout:
  37     """The layout of the dash app and the callbacks.
  38     """
  39
  40     def __init__(self, app: Flask, html_layout_file: str, data_spec_file: str,
  41         tooltip_file: str) -> None:
  42         """Initialization:
  43         - save the input parameters,
  44         - read and pre-process the data,
  45         - prepare data for the control panel,
  46         - read HTML layout file,
  47         - read tooltips from the tooltip file.
  48
  49         :param app: Flask application running the dash application.
  50         :param html_layout_file: Path and name of the file specifying the HTML
  51             layout of the dash application.
  52         :param data_spec_file: Path and name of the file specifying the data to
  53             be read from parquets for this application.
  54         :param tooltip_file: Path and name of the yaml file specifying the
  55             tooltips.
  56         :type app: Flask
  57         :type html_layout_file: str
  58         :type data_spec_file: str
  59         :type tooltip_file: str
  60         """
  61
  62         # Inputs
  63         self._app = app
  64         self._html_layout_file = html_layout_file
  65         self._data_spec_file = data_spec_file
  66         self._tooltip_file = tooltip_file
  67
  68         # Read the data:
  69         data_stats, data_mrr, data_ndrpdr = Data(
  70             data_spec_file=self._data_spec_file,
  71             debug=True
  72         ).read_stats(days=C.NEWS_TIME_PERIOD)
  73
  74         df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
  75
  76         # Prepare information for the control panel:
  77         self._jobs = sorted(list(df_tst_info["job"].unique()))
  78         d_job_info = {
  79             "job": list(),
  80             "dut": list(),
  81             "ttype": list(),
  82             "cadence": list(),
  83             "tbed": list()
  84         }
  85         for job in self._jobs:
  86             lst_job = job.split("-")
  87             d_job_info["job"].append(job)
  88             d_job_info["dut"].append(lst_job[1])
  89             d_job_info["ttype"].append(lst_job[3])
  90             d_job_info["cadence"].append(lst_job[4])
  91             d_job_info["tbed"].append("-".join(lst_job[-2:]))
  92         self.job_info = pd.DataFrame.from_dict(d_job_info)
  93
  94         # Pre-process the data:
  95
  96         def _create_test_name(test: str) -> str:
  97             lst_tst = test.split(".")
  98             suite = lst_tst[-2].replace("2n1l-", "").replace("1n1l-", "").\
  99                 replace("2n-", "")
 100             return f"{suite.split('-')[0]}-{lst_tst[-1]}"
 101
 102         def _get_rindex(array: list, itm: any) -> int:
 103             return len(array) - 1 - array[::-1].index(itm)
 104
 105         tst_info = {
 106             "job": list(),
 107             "build": list(),
 108             "start": list(),
 109             "dut_type": list(),
 110             "dut_version": list(),
 111             "hosts": list(),
 112             "failed": list(),
 113             "regressions": list(),
 114             "progressions": list()
 115         }
 116         for job in self._jobs:
 117             # Create lists of failed tests:
 118             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
 119             last_build = str(max(pd.to_numeric(df_job["build"].unique())))
 120             df_build = df_job.loc[(df_job["build"] == last_build)]
 121             tst_info["job"].append(job)
 122             tst_info["build"].append(last_build)
 123             tst_info["start"].append(data_stats.loc[
 124                 (data_stats["job"] == job) &
 125                 (data_stats["build"] == last_build)
 126             ]["start_time"].iloc[-1].strftime('%Y-%m-%d %H:%M'))
 127             tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
 128             tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
 129             tst_info["hosts"].append(df_build["hosts"].iloc[-1])
 130             failed_tests = df_build.loc[(df_build["passed"] == False)]\
 131                 ["test_id"].to_list()
 132             l_failed = list()
 133             try:
 134                 for tst in failed_tests:
 135                     l_failed.append(_create_test_name(tst))
 136             except KeyError:
 137                 l_failed = list()
 138             tst_info["failed"].append(sorted(l_failed))
 139
 140             # Create lists of regressions and progressions:
 141             l_reg = list()
 142             l_prog = list()
 143
 144             tests = df_job["test_id"].unique()
 145             for test in tests:
 146                 tst_data = df_job.loc[df_job["test_id"] == test].sort_values(
 147                     by="start_time", ignore_index=True)
 148                 x_axis = tst_data["start_time"].tolist()
 149                 if "-ndrpdr" in test:
 150                     tst_data = tst_data.dropna(
 151                         subset=["result_pdr_lower_rate_value", ]
 152                     )
 153                     if tst_data.empty:
 154                         continue
 155                     try:
 156                         anomalies, _, _ = classify_anomalies({
 157                             k: v for k, v in zip(
 158                                 x_axis,
 159                                 tst_data["result_ndr_lower_rate_value"].tolist()
 160                             )
 161                         })
 162                     except ValueError:
 163                         continue
 164                     if "progression" in anomalies:
 165                         l_prog.append((
 166                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 167                             x_axis[_get_rindex(anomalies, "progression")]
 168                         ))
 169                     if "regression" in anomalies:
 170                         l_reg.append((
 171                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 172                             x_axis[_get_rindex(anomalies, "regression")]
 173                         ))
 174                     try:
 175                         anomalies, _, _ = classify_anomalies({
 176                             k: v for k, v in zip(
 177                                 x_axis,
 178                                 tst_data["result_pdr_lower_rate_value"].tolist()
 179                             )
 180                         })
 181                     except ValueError:
 182                         continue
 183                     if "progression" in anomalies:
 184                         l_prog.append((
 185                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 186                             x_axis[_get_rindex(anomalies, "progression")]
 187                         ))
 188                     if "regression" in anomalies:
 189                         l_reg.append((
 190                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 191                             x_axis[_get_rindex(anomalies, "regression")]
 192                         ))
 193                 else:  # mrr
 194                     tst_data = tst_data.dropna(
 195                         subset=["result_receive_rate_rate_avg", ]
 196                     )
 197                     if tst_data.empty:
 198                         continue
 199                     try:
 200                         anomalies, _, _ = classify_anomalies({
 201                             k: v for k, v in zip(
 202                                 x_axis,
 203                                 tst_data["result_receive_rate_rate_avg"].\
 204                                     tolist()
 205                             )
 206                         })
 207                     except ValueError:
 208                         continue
 209                     if "progression" in anomalies:
 210                         l_prog.append((
 211                             _create_test_name(test),
 212                             x_axis[_get_rindex(anomalies, "progression")]
 213                         ))
 214                     if "regression" in anomalies:
 215                         l_reg.append((
 216                             _create_test_name(test),
 217                             x_axis[_get_rindex(anomalies, "regression")]
 218                         ))
 219
 220             tst_info["regressions"].append(
 221                 sorted(l_reg, key=lambda k: k[1], reverse=True))
 222             tst_info["progressions"].append(
 223                 sorted(l_prog, key=lambda k: k[1], reverse=True))
 224
 225         self._data = pd.DataFrame.from_dict(tst_info)
 226
 227         # Read from files:
 228         self._html_layout = str()
 229         self._tooltips = dict()
 230
 231         try:
 232             with open(self._html_layout_file, "r") as file_read:
 233                 self._html_layout = file_read.read().\
 234                     replace("_title_", C.NEWS_TITLE)
 235         except IOError as err:
 236             raise RuntimeError(
 237                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 238             )
 239
 240         try:
 241             with open(self._tooltip_file, "r") as file_read:
 242                 self._tooltips = load(file_read, Loader=FullLoader)
 243         except IOError as err:
 244             logging.warning(
 245                 f"Not possible to open the file {self._tooltip_file}\n{err}"
 246             )
 247         except YAMLError as err:
 248             logging.warning(
 249                 f"An error occurred while parsing the specification file "
 250                 f"{self._tooltip_file}\n{err}"
 251             )
 252
 253         self._default_period = C.NEWS_SHORT
 254         self._default_active = (False, True, False)
 255         self._default_table = \
 256             table_summary(self._data, self._jobs, self._default_period)
 257
 258         # Callbacks:
 259         if self._app is not None and hasattr(self, 'callbacks'):
 260             self.callbacks(self._app)
 261
 262     @property
 263     def html_layout(self) -> dict:
 264         return self._html_layout
 265
 266     def add_content(self):
 267         """Top level method which generated the web page.
 268
 269         It generates:
 270         - Store for user input data,
 271         - Navigation bar,
 272         - Main area with control panel and ploting area.
 273
 274         If no HTML layout is provided, an error message is displayed instead.
 275
 276         :returns: The HTML div with the whole page.
 277         :rtype: html.Div
 278         """
 279
 280         if self.html_layout:
 281             return html.Div(
 282                 id="div-main",
 283                 className="small",
 284                 children=[
 285                     dcc.Location(id="url", refresh=False),
 286                     dbc.Row(
 287                         id="row-navbar",
 288                         class_name="g-0",
 289                         children=[
 290                             self._add_navbar(),
 291                         ]
 292                     ),
 293                     dbc.Row(
 294                         id="row-main",
 295                         class_name="g-0",
 296                         children=[
 297                             self._add_ctrl_col(),
 298                             self._add_plotting_col(),
 299                         ]
 300                     )
 301                 ]
 302             )
 303         else:
 304             return html.Div(
 305                 id="div-main-error",
 306                 children=[
 307                     dbc.Alert(
 308                         [
 309                             "An Error Occured",
 310                         ],
 311                         color="danger",
 312                     ),
 313                 ]
 314             )
 315
 316     def _add_navbar(self):
 317         """Add nav element with navigation panel. It is placed on the top.
 318
 319         :returns: Navigation bar.
 320         :rtype: dbc.NavbarSimple
 321         """
 322
 323         return dbc.NavbarSimple(
 324             id="navbarsimple-main",
 325             children=[
 326                 dbc.NavItem(
 327                     dbc.NavLink(
 328                         C.NEWS_TITLE,
 329                         disabled=True,
 330                         external_link=True,
 331                         href="#"
 332                     )
 333                 )
 334             ],
 335             brand=C.BRAND,
 336             brand_href="/",
 337             brand_external_link=True,
 338             class_name="p-2",
 339             fluid=True,
 340         )
 341
 342     def _add_ctrl_col(self) -> dbc.Col:
 343         """Add column with control panel. It is placed on the left side.
 344
 345         :returns: Column with the control panel.
 346         :rtype: dbc.Col
 347         """
 348         return dbc.Col([
 349             html.Div(
 350                 children=self._add_ctrl_panel(),
 351                 className="sticky-top"
 352             )
 353         ])
 354
 355     def _add_plotting_col(self) -> dbc.Col:
 356         """Add column with tables. It is placed on the right side.
 357
 358         :returns: Column with tables.
 359         :rtype: dbc.Col
 360         """
 361
 362         return dbc.Col(
 363             id="col-plotting-area",
 364             children=[
 365                 dcc.Loading(
 366                     children=[
 367                         dbc.Row(  # Failed tests
 368                             id="row-table",
 369                             class_name="g-0 p-2",
 370                             children=self._default_table
 371                         ),
 372                         dbc.Row(
 373                             class_name="g-0 p-2",
 374                             align="center",
 375                             justify="start",
 376                             children=[
 377                                 dbc.InputGroup(
 378                                     class_name="me-1",
 379                                     children=[
 380                                         dbc.InputGroupText(
 381                                             style=C.URL_STYLE,
 382                                             children=show_tooltip(
 383                                                 self._tooltips,
 384                                                 "help-url", "URL",
 385                                                 "input-url"
 386                                             )
 387                                         ),
 388                                         dbc.Input(
 389                                             id="input-url",
 390                                             readonly=True,
 391                                             type="url",
 392                                             style=C.URL_STYLE,
 393                                             value=""
 394                                         )
 395                                     ]
 396                                 )
 397                             ]
 398                         )
 399                     ]
 400                 )
 401             ],
 402             width=9,
 403         )
 404
 405     def _add_ctrl_panel(self) -> dbc.Row:
 406         """Add control panel.
 407
 408         :returns: Control panel.
 409         :rtype: dbc.Row
 410         """
 411         return [
 412             dbc.Label(
 413                 class_name="g-0 p-1",
 414                 children=show_tooltip(self._tooltips,
 415                     "help-summary-period", "Window")
 416             ),
 417             dbc.Row(
 418                 class_name="g-0 p-1",
 419                 children=[
 420                     dbc.ButtonGroup(
 421                         id="bg-time-period",
 422                         children=[
 423                             dbc.Button(
 424                                 id="period-last",
 425                                 children="Last Run",
 426                                 className="me-1",
 427                                 outline=True,
 428                                 color="info"
 429                             ),
 430                             dbc.Button(
 431                                 id="period-short",
 432                                 children=f"Last {C.NEWS_SHORT} Runs",
 433                                 className="me-1",
 434                                 outline=True,
 435                                 active=True,
 436                                 color="info"
 437                             ),
 438                             dbc.Button(
 439                                 id="period-long",
 440                                 children="All Runs",
 441                                 className="me-1",
 442                                 outline=True,
 443                                 color="info"
 444                             )
 445                         ]
 446                     )
 447                 ]
 448             )
 449         ]
 450
 451     def callbacks(self, app):
 452         """Callbacks for the whole application.
 453
 454         :param app: The application.
 455         :type app: Flask
 456         """
 457
 458         @app.callback(
 459             Output("row-table", "children"),
 460             Output("input-url", "value"),
 461             Output("period-last", "active"),
 462             Output("period-short", "active"),
 463             Output("period-long", "active"),
 464             Input("period-last", "n_clicks"),
 465             Input("period-short", "n_clicks"),
 466             Input("period-long", "n_clicks"),
 467             Input("url", "href")
 468         )
 469         def _update_application(btn_last: int, btn_short: int, btn_long: int,
 470             href: str) -> tuple:
 471             """Update the application when the event is detected.
 472
 473             :returns: New values for web page elements.
 474             :rtype: tuple
 475             """
 476
 477             _, _, _ = btn_last, btn_short, btn_long
 478
 479             periods = {
 480                 "period-last": C.NEWS_LAST,
 481                 "period-short": C.NEWS_SHORT,
 482                 "period-long": C.NEWS_LONG
 483             }
 484             actives = {
 485                 "period-last": (True, False, False),
 486                 "period-short": (False, True, False),
 487                 "period-long": (False, False, True)
 488             }
 489
 490             # Parse the url:
 491             parsed_url = url_decode(href)
 492             if parsed_url:
 493                 url_params = parsed_url["params"]
 494             else:
 495                 url_params = None
 496
 497             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 498             if trigger_id == "url" and url_params:
 499                 trigger_id = url_params.get("period", list())[0]
 500
 501             period = periods.get(trigger_id, self._default_period)
 502             active = actives.get(trigger_id, self._default_active)
 503
 504             ret_val = [
 505                 table_summary(self._data, self._jobs, period),
 506                 gen_new_url(parsed_url, {"period": trigger_id})
 507             ]
 508             ret_val.extend(active)
 509             return ret_val