csit.infra.dash/app/cdash/news/layout.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import logging
  18 import pandas as pd
  19 import dash_bootstrap_components as dbc
  20
  21 from flask import Flask
  22 from dash import dcc
  23 from dash import html
  24 from dash import callback_context
  25 from dash import Input, Output, State
  26 from yaml import load, FullLoader, YAMLError
  27
  28 from ..data.data import Data
  29 from ..utils.constants import Constants as C
  30 from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url
  31 from ..utils.url_processing import url_decode
  32 from ..data.data import Data
  33 from .tables import table_summary
  34
  35
  36 class Layout:
  37     """The layout of the dash app and the callbacks.
  38     """
  39
  40     def __init__(self, app: Flask, html_layout_file: str, data_spec_file: str,
  41         tooltip_file: str) -> None:
  42         """Initialization:
  43         - save the input parameters,
  44         - read and pre-process the data,
  45         - prepare data for the control panel,
  46         - read HTML layout file,
  47         - read tooltips from the tooltip file.
  48
  49         :param app: Flask application running the dash application.
  50         :param html_layout_file: Path and name of the file specifying the HTML
  51             layout of the dash application.
  52         :param data_spec_file: Path and name of the file specifying the data to
  53             be read from parquets for this application.
  54         :param tooltip_file: Path and name of the yaml file specifying the
  55             tooltips.
  56         :type app: Flask
  57         :type html_layout_file: str
  58         :type data_spec_file: str
  59         :type tooltip_file: str
  60         """
  61
  62         # Inputs
  63         self._app = app
  64         self._html_layout_file = html_layout_file
  65         self._data_spec_file = data_spec_file
  66         self._tooltip_file = tooltip_file
  67
  68         # Read the data:
  69         data_stats, data_mrr, data_ndrpdr = Data(
  70             data_spec_file=self._data_spec_file,
  71             debug=True
  72         ).read_stats(days=C.NEWS_TIME_PERIOD)
  73
  74         df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
  75
  76         # Prepare information for the control panel:
  77         self._jobs = sorted(list(df_tst_info["job"].unique()))
  78         d_job_info = {
  79             "job": list(),
  80             "dut": list(),
  81             "ttype": list(),
  82             "cadence": list(),
  83             "tbed": list()
  84         }
  85         for job in self._jobs:
  86             lst_job = job.split("-")
  87             d_job_info["job"].append(job)
  88             d_job_info["dut"].append(lst_job[1])
  89             d_job_info["ttype"].append(lst_job[3])
  90             d_job_info["cadence"].append(lst_job[4])
  91             d_job_info["tbed"].append("-".join(lst_job[-2:]))
  92         self.job_info = pd.DataFrame.from_dict(d_job_info)
  93
  94         # Pre-process the data:
  95
  96         def _create_test_name(test: str) -> str:
  97             lst_tst = test.split(".")
  98             suite = lst_tst[-2].replace("2n1l-", "").replace("1n1l-", "").\
  99                 replace("2n-", "")
 100             return f"{suite.split('-')[0]}-{lst_tst[-1]}"
 101
 102         def _get_rindex(array: list, itm: any) -> int:
 103             return len(array) - 1 - array[::-1].index(itm)
 104
 105         tst_info = {
 106             "job": list(),
 107             "build": list(),
 108             "start": list(),
 109             "dut_type": list(),
 110             "dut_version": list(),
 111             "hosts": list(),
 112             "failed": list(),
 113             "regressions": list(),
 114             "progressions": list()
 115         }
 116         for job in self._jobs:
 117             # Create lists of failed tests:
 118             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
 119             last_build = str(max(pd.to_numeric(df_job["build"].unique())))
 120             df_build = df_job.loc[(df_job["build"] == last_build)]
 121             tst_info["job"].append(job)
 122             tst_info["build"].append(last_build)
 123             tst_info["start"].append(data_stats.loc[
 124                 (data_stats["job"] == job) &
 125                 (data_stats["build"] == last_build)
 126             ]["start_time"].iloc[-1].strftime('%Y-%m-%d %H:%M'))
 127             tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
 128             tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
 129             tst_info["hosts"].append(df_build["hosts"].iloc[-1])
 130             failed_tests = df_build.loc[(df_build["passed"] == False)]\
 131                 ["test_id"].to_list()
 132             l_failed = list()
 133             try:
 134                 for tst in failed_tests:
 135                     l_failed.append(_create_test_name(tst))
 136             except KeyError:
 137                 l_failed = list()
 138             tst_info["failed"].append(sorted(l_failed))
 139
 140             # Create lists of regressions and progressions:
 141             l_reg = list()
 142             l_prog = list()
 143
 144             tests = df_job["test_id"].unique()
 145             for test in tests:
 146                 tst_data = df_job.loc[df_job["test_id"] == test].sort_values(
 147                     by="start_time", ignore_index=True)
 148                 x_axis = tst_data["start_time"].tolist()
 149                 if "-ndrpdr" in test:
 150                     tst_data = tst_data.dropna(
 151                         subset=["result_pdr_lower_rate_value", ]
 152                     )
 153                     if tst_data.empty:
 154                         continue
 155                     try:
 156                         anomalies, _, _ = classify_anomalies({
 157                             k: v for k, v in zip(
 158                                 x_axis,
 159                                 tst_data["result_ndr_lower_rate_value"].tolist()
 160                             )
 161                         })
 162                     except ValueError:
 163                         continue
 164                     if "progression" in anomalies:
 165                         l_prog.append((
 166                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 167                             x_axis[_get_rindex(anomalies, "progression")]
 168                         ))
 169                     if "regression" in anomalies:
 170                         l_reg.append((
 171                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 172                             x_axis[_get_rindex(anomalies, "regression")]
 173                         ))
 174                     try:
 175                         anomalies, _, _ = classify_anomalies({
 176                             k: v for k, v in zip(
 177                                 x_axis,
 178                                 tst_data["result_pdr_lower_rate_value"].tolist()
 179                             )
 180                         })
 181                     except ValueError:
 182                         continue
 183                     if "progression" in anomalies:
 184                         l_prog.append((
 185                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 186                             x_axis[_get_rindex(anomalies, "progression")]
 187                         ))
 188                     if "regression" in anomalies:
 189                         l_reg.append((
 190                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 191                             x_axis[_get_rindex(anomalies, "regression")]
 192                         ))
 193                 else:  # mrr
 194                     tst_data = tst_data.dropna(
 195                         subset=["result_receive_rate_rate_avg", ]
 196                     )
 197                     if tst_data.empty:
 198                         continue
 199                     try:
 200                         anomalies, _, _ = classify_anomalies({
 201                             k: v for k, v in zip(
 202                                 x_axis,
 203                                 tst_data["result_receive_rate_rate_avg"].\
 204                                     tolist()
 205                             )
 206                         })
 207                     except ValueError:
 208                         continue
 209                     if "progression" in anomalies:
 210                         l_prog.append((
 211                             _create_test_name(test),
 212                             x_axis[_get_rindex(anomalies, "progression")]
 213                         ))
 214                     if "regression" in anomalies:
 215                         l_reg.append((
 216                             _create_test_name(test),
 217                             x_axis[_get_rindex(anomalies, "regression")]
 218                         ))
 219
 220             tst_info["regressions"].append(
 221                 sorted(l_reg, key=lambda k: k[1], reverse=True))
 222             tst_info["progressions"].append(
 223                 sorted(l_prog, key=lambda k: k[1], reverse=True))
 224
 225         self._data = pd.DataFrame.from_dict(tst_info)
 226
 227         # Read from files:
 228         self._html_layout = str()
 229         self._tooltips = dict()
 230
 231         try:
 232             with open(self._html_layout_file, "r") as file_read:
 233                 self._html_layout = file_read.read()
 234         except IOError as err:
 235             raise RuntimeError(
 236                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 237             )
 238
 239         try:
 240             with open(self._tooltip_file, "r") as file_read:
 241                 self._tooltips = load(file_read, Loader=FullLoader)
 242         except IOError as err:
 243             logging.warning(
 244                 f"Not possible to open the file {self._tooltip_file}\n{err}"
 245             )
 246         except YAMLError as err:
 247             logging.warning(
 248                 f"An error occurred while parsing the specification file "
 249                 f"{self._tooltip_file}\n{err}"
 250             )
 251
 252         self._default_period = C.NEWS_SHORT
 253         self._default_active = (False, True, False)
 254
 255         # Callbacks:
 256         if self._app is not None and hasattr(self, 'callbacks'):
 257             self.callbacks(self._app)
 258
 259     @property
 260     def html_layout(self) -> dict:
 261         return self._html_layout
 262
 263     def add_content(self):
 264         """Top level method which generated the web page.
 265
 266         It generates:
 267         - Store for user input data,
 268         - Navigation bar,
 269         - Main area with control panel and ploting area.
 270
 271         If no HTML layout is provided, an error message is displayed instead.
 272
 273         :returns: The HTML div with the whole page.
 274         :rtype: html.Div
 275         """
 276
 277         if self.html_layout:
 278             return html.Div(
 279                 id="div-main",
 280                 className="small",
 281                 children=[
 282                     dcc.Location(id="url", refresh=False),
 283                     dbc.Row(
 284                         id="row-navbar",
 285                         class_name="g-0",
 286                         children=[
 287                             self._add_navbar()
 288                         ]
 289                     ),
 290                     dbc.Row(
 291                         id="row-main",
 292                         class_name="g-0",
 293                         children=[
 294                             self._add_ctrl_col(),
 295                             self._add_plotting_col()
 296                         ]
 297                     )
 298                 ]
 299             )
 300         else:
 301             return html.Div(
 302                 id="div-main-error",
 303                 children=[
 304                     dbc.Alert(
 305                         [
 306                             "An Error Occured"
 307                         ],
 308                         color="danger"
 309                     )
 310                 ]
 311             )
 312
 313     def _add_navbar(self):
 314         """Add nav element with navigation panel. It is placed on the top.
 315
 316         :returns: Navigation bar.
 317         :rtype: dbc.NavbarSimple
 318         """
 319
 320         return dbc.NavbarSimple(
 321             id="navbarsimple-main",
 322             children=[
 323                 dbc.NavItem(
 324                     dbc.NavLink(
 325                         C.NEWS_TITLE,
 326                         disabled=True,
 327                         external_link=True,
 328                         href="#"
 329                     )
 330                 )
 331             ],
 332             brand=C.BRAND,
 333             brand_href="/",
 334             brand_external_link=True,
 335             class_name="p-2",
 336             fluid=True
 337         )
 338
 339     def _add_ctrl_col(self) -> dbc.Col:
 340         """Add column with control panel. It is placed on the left side.
 341
 342         :returns: Column with the control panel.
 343         :rtype: dbc.Col
 344         """
 345         return dbc.Col([
 346             html.Div(
 347                 children=self._add_ctrl_panel(),
 348                 className="sticky-top"
 349             )
 350         ])
 351
 352     def _add_plotting_col(self) -> dbc.Col:
 353         """Add column with tables. It is placed on the right side.
 354
 355         :returns: Column with tables.
 356         :rtype: dbc.Col
 357         """
 358         return dbc.Col(
 359             id="col-plotting-area",
 360             children=[
 361                 dcc.Loading(
 362                     children=[
 363                         dbc.Row(
 364                             id="plotting-area",
 365                             class_name="g-0 p-0",
 366                             children=[
 367                                 C.PLACEHOLDER
 368                             ]
 369                         )
 370                     ]
 371                 )
 372             ],
 373             width=9
 374         )
 375
 376     def _add_ctrl_panel(self) -> list:
 377         """Add control panel.
 378
 379         :returns: Control panel.
 380         :rtype: list
 381         """
 382         return [
 383             dbc.Row(
 384                 class_name="g-0 p-1",
 385                 children=[
 386                     dbc.ButtonGroup(
 387                         id="bg-time-period",
 388                         children=[
 389                             dbc.Button(
 390                                 id="period-last",
 391                                 children="Last Run",
 392                                 className="me-1",
 393                                 outline=True,
 394                                 color="info"
 395                             ),
 396                             dbc.Button(
 397                                 id="period-short",
 398                                 children=f"Last {C.NEWS_SHORT} Runs",
 399                                 className="me-1",
 400                                 outline=True,
 401                                 active=True,
 402                                 color="info"
 403                             ),
 404                             dbc.Button(
 405                                 id="period-long",
 406                                 children="All Runs",
 407                                 className="me-1",
 408                                 outline=True,
 409                                 color="info"
 410                             )
 411                         ]
 412                     )
 413                 ]
 414             )
 415         ]
 416
 417     def _get_plotting_area(
 418             self,
 419             period: int,
 420             url: str
 421         ) -> list:
 422         """Generate the plotting area with all its content.
 423
 424         :param period: The time period for summary tables.
 425         :param url: URL to be displayed in the modal window.
 426         :type period: int
 427         :type url: str
 428         :returns: The content of the plotting area.
 429         :rtype: list
 430         """
 431         return [
 432             dbc.Row(
 433                 id="row-table",
 434                 class_name="g-0 p-1",
 435                 children=table_summary(self._data, self._jobs, period)
 436             ),
 437             dbc.Row(
 438                 [
 439                     dbc.Col([html.Div(
 440                         [
 441                             dbc.Button(
 442                                 id="plot-btn-url",
 443                                 children="Show URL",
 444                                 class_name="me-1",
 445                                 color="info",
 446                                 style={
 447                                     "text-transform": "none",
 448                                     "padding": "0rem 1rem"
 449                                 }
 450                             ),
 451                             dbc.Modal(
 452                                 [
 453                                     dbc.ModalHeader(dbc.ModalTitle("URL")),
 454                                     dbc.ModalBody(url)
 455                                 ],
 456                                 id="plot-mod-url",
 457                                 size="xl",
 458                                 is_open=False,
 459                                 scrollable=True
 460                             )
 461                         ],
 462                         className=\
 463                             "d-grid gap-0 d-md-flex justify-content-md-end"
 464                     )])
 465                 ],
 466                 class_name="g-0 p-0"
 467             )
 468         ]
 469
 470     def callbacks(self, app):
 471         """Callbacks for the whole application.
 472
 473         :param app: The application.
 474         :type app: Flask
 475         """
 476
 477         @app.callback(
 478             Output("plotting-area", "children"),
 479             Output("period-last", "active"),
 480             Output("period-short", "active"),
 481             Output("period-long", "active"),
 482             Input("url", "href"),
 483             Input("period-last", "n_clicks"),
 484             Input("period-short", "n_clicks"),
 485             Input("period-long", "n_clicks")
 486         )
 487         def _update_application(href: str, *_) -> tuple:
 488             """Update the application when the event is detected.
 489
 490             :returns: New values for web page elements.
 491             :rtype: tuple
 492             """
 493
 494             periods = {
 495                 "period-last": C.NEWS_LAST,
 496                 "period-short": C.NEWS_SHORT,
 497                 "period-long": C.NEWS_LONG
 498             }
 499             actives = {
 500                 "period-last": (True, False, False),
 501                 "period-short": (False, True, False),
 502                 "period-long": (False, False, True)
 503             }
 504
 505             # Parse the url:
 506             parsed_url = url_decode(href)
 507             if parsed_url:
 508                 url_params = parsed_url["params"]
 509             else:
 510                 url_params = None
 511
 512             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 513             if trigger_id == "url" and url_params:
 514                 trigger_id = url_params.get("period", list())[0]
 515
 516             ret_val = [
 517                 self._get_plotting_area(
 518                     periods.get(trigger_id, self._default_period),
 519                     gen_new_url(parsed_url, {"period": trigger_id})
 520                 )
 521             ]
 522             ret_val.extend(actives.get(trigger_id, self._default_active))
 523             return ret_val
 524
 525         @app.callback(
 526             Output("plot-mod-url", "is_open"),
 527             [Input("plot-btn-url", "n_clicks")],
 528             [State("plot-mod-url", "is_open")],
 529         )
 530         def toggle_plot_mod_url(n, is_open):
 531             """Toggle the modal window with url.
 532             """
 533             if n:
 534                 return not is_open
 535             return is_open