csit.infra.dash/app/cdash/news/layout.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import pandas as pd
  18 import dash_bootstrap_components as dbc
  19
  20 from flask import Flask
  21 from dash import dcc
  22 from dash import html
  23 from dash import callback_context
  24 from dash import Input, Output, State
  25
  26 from ..utils.constants import Constants as C
  27 from ..utils.utils import classify_anomalies, gen_new_url
  28 from ..utils.url_processing import url_decode
  29 from .tables import table_summary
  30
  31
  32 class Layout:
  33     """The layout of the dash app and the callbacks.
  34     """
  35
  36     def __init__(
  37             self,
  38             app: Flask,
  39             data_stats: pd.DataFrame,
  40             data_trending: pd.DataFrame,
  41             html_layout_file: str
  42         ) -> None:
  43         """Initialization:
  44         - save the input parameters,
  45         - read and pre-process the data,
  46         - prepare data for the control panel,
  47         - read HTML layout file,
  48         - read tooltips from the tooltip file.
  49
  50         :param app: Flask application running the dash application.
  51         :param data_stats: Pandas dataframe with staistical data.
  52         :param data_trending: Pandas dataframe with trending data.
  53         :param html_layout_file: Path and name of the file specifying the HTML
  54             layout of the dash application.
  55         :type app: Flask
  56         :type data_stats: pandas.DataFrame
  57         :type data_trending: pandas.DataFrame
  58         :type html_layout_file: str
  59         """
  60
  61         # Inputs
  62         self._app = app
  63         self._html_layout_file = html_layout_file
  64
  65         # Prepare information for the control panel:
  66         self._jobs = sorted(list(data_trending["job"].unique()))
  67         d_job_info = {
  68             "job": list(),
  69             "dut": list(),
  70             "ttype": list(),
  71             "cadence": list(),
  72             "tbed": list()
  73         }
  74         for job in self._jobs:
  75             lst_job = job.split("-")
  76             d_job_info["job"].append(job)
  77             d_job_info["dut"].append(lst_job[1])
  78             d_job_info["ttype"].append(lst_job[3])
  79             d_job_info["cadence"].append(lst_job[4])
  80             d_job_info["tbed"].append("-".join(lst_job[-2:]))
  81         self.job_info = pd.DataFrame.from_dict(d_job_info)
  82
  83         # Pre-process the data:
  84
  85         def _create_test_name(test: str) -> str:
  86             lst_tst = test.split(".")
  87             suite = lst_tst[-2].replace("2n1l-", "").replace("1n1l-", "").\
  88                 replace("2n-", "")
  89             return f"{suite.split('-')[0]}-{lst_tst[-1]}"
  90
  91         def _get_rindex(array: list, itm: any) -> int:
  92             return len(array) - 1 - array[::-1].index(itm)
  93
  94         tst_info = {
  95             "job": list(),
  96             "build": list(),
  97             "start": list(),
  98             "dut_type": list(),
  99             "dut_version": list(),
 100             "hosts": list(),
 101             "failed": list(),
 102             "regressions": list(),
 103             "progressions": list()
 104         }
 105         for job in self._jobs:
 106             # Create lists of failed tests:
 107             df_job = data_trending.loc[(data_trending["job"] == job)]
 108             last_build = str(max(pd.to_numeric(df_job["build"].unique())))
 109             df_build = df_job.loc[(df_job["build"] == last_build)]
 110             tst_info["job"].append(job)
 111             tst_info["build"].append(last_build)
 112             tst_info["start"].append(data_stats.loc[
 113                 (data_stats["job"] == job) &
 114                 (data_stats["build"] == last_build)
 115             ]["start_time"].iloc[-1].strftime('%Y-%m-%d %H:%M'))
 116             tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
 117             tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
 118             tst_info["hosts"].append(df_build["hosts"].iloc[-1])
 119             failed_tests = df_build.loc[(df_build["passed"] == False)]\
 120                 ["test_id"].to_list()
 121             l_failed = list()
 122             try:
 123                 for tst in failed_tests:
 124                     l_failed.append(_create_test_name(tst))
 125             except KeyError:
 126                 l_failed = list()
 127             tst_info["failed"].append(sorted(l_failed))
 128
 129             # Create lists of regressions and progressions:
 130             l_reg = list()
 131             l_prog = list()
 132
 133             tests = df_job["test_id"].unique()
 134             for test in tests:
 135                 tst_data = df_job.loc[df_job["test_id"] == test].sort_values(
 136                     by="start_time", ignore_index=True)
 137                 x_axis = tst_data["start_time"].tolist()
 138                 if "-ndrpdr" in test:
 139                     tst_data = tst_data.dropna(
 140                         subset=["result_pdr_lower_rate_value", ]
 141                     )
 142                     if tst_data.empty:
 143                         continue
 144                     try:
 145                         anomalies, _, _ = classify_anomalies({
 146                             k: v for k, v in zip(
 147                                 x_axis,
 148                                 tst_data["result_ndr_lower_rate_value"].tolist()
 149                             )
 150                         })
 151                     except ValueError:
 152                         continue
 153                     if "progression" in anomalies:
 154                         l_prog.append((
 155                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 156                             x_axis[_get_rindex(anomalies, "progression")]
 157                         ))
 158                     if "regression" in anomalies:
 159                         l_reg.append((
 160                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 161                             x_axis[_get_rindex(anomalies, "regression")]
 162                         ))
 163                     try:
 164                         anomalies, _, _ = classify_anomalies({
 165                             k: v for k, v in zip(
 166                                 x_axis,
 167                                 tst_data["result_pdr_lower_rate_value"].tolist()
 168                             )
 169                         })
 170                     except ValueError:
 171                         continue
 172                     if "progression" in anomalies:
 173                         l_prog.append((
 174                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 175                             x_axis[_get_rindex(anomalies, "progression")]
 176                         ))
 177                     if "regression" in anomalies:
 178                         l_reg.append((
 179                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 180                             x_axis[_get_rindex(anomalies, "regression")]
 181                         ))
 182                 else:  # mrr
 183                     tst_data = tst_data.dropna(
 184                         subset=["result_receive_rate_rate_avg", ]
 185                     )
 186                     if tst_data.empty:
 187                         continue
 188                     try:
 189                         anomalies, _, _ = classify_anomalies({
 190                             k: v for k, v in zip(
 191                                 x_axis,
 192                                 tst_data["result_receive_rate_rate_avg"].\
 193                                     tolist()
 194                             )
 195                         })
 196                     except ValueError:
 197                         continue
 198                     if "progression" in anomalies:
 199                         l_prog.append((
 200                             _create_test_name(test),
 201                             x_axis[_get_rindex(anomalies, "progression")]
 202                         ))
 203                     if "regression" in anomalies:
 204                         l_reg.append((
 205                             _create_test_name(test),
 206                             x_axis[_get_rindex(anomalies, "regression")]
 207                         ))
 208
 209             tst_info["regressions"].append(
 210                 sorted(l_reg, key=lambda k: k[1], reverse=True))
 211             tst_info["progressions"].append(
 212                 sorted(l_prog, key=lambda k: k[1], reverse=True))
 213
 214         self._data = pd.DataFrame.from_dict(tst_info)
 215
 216         # Read from files:
 217         self._html_layout = str()
 218
 219         try:
 220             with open(self._html_layout_file, "r") as file_read:
 221                 self._html_layout = file_read.read()
 222         except IOError as err:
 223             raise RuntimeError(
 224                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 225             )
 226
 227         self._default_period = C.NEWS_SHORT
 228         self._default_active = (False, True, False)
 229
 230         # Callbacks:
 231         if self._app is not None and hasattr(self, 'callbacks'):
 232             self.callbacks(self._app)
 233
 234     @property
 235     def html_layout(self) -> dict:
 236         return self._html_layout
 237
 238     def add_content(self):
 239         """Top level method which generated the web page.
 240
 241         It generates:
 242         - Store for user input data,
 243         - Navigation bar,
 244         - Main area with control panel and ploting area.
 245
 246         If no HTML layout is provided, an error message is displayed instead.
 247
 248         :returns: The HTML div with the whole page.
 249         :rtype: html.Div
 250         """
 251
 252         if self.html_layout:
 253             return html.Div(
 254                 id="div-main",
 255                 className="small",
 256                 children=[
 257                     dcc.Location(id="url", refresh=False),
 258                     dbc.Row(
 259                         id="row-navbar",
 260                         class_name="g-0",
 261                         children=[
 262                             self._add_navbar()
 263                         ]
 264                     ),
 265                     dbc.Row(
 266                         id="row-main",
 267                         class_name="g-0",
 268                         children=[
 269                             self._add_ctrl_col(),
 270                             self._add_plotting_col()
 271                         ]
 272                     )
 273                 ]
 274             )
 275         else:
 276             return html.Div(
 277                 id="div-main-error",
 278                 children=[
 279                     dbc.Alert(
 280                         [
 281                             "An Error Occured"
 282                         ],
 283                         color="danger"
 284                     )
 285                 ]
 286             )
 287
 288     def _add_navbar(self):
 289         """Add nav element with navigation panel. It is placed on the top.
 290
 291         :returns: Navigation bar.
 292         :rtype: dbc.NavbarSimple
 293         """
 294
 295         return dbc.NavbarSimple(
 296             id="navbarsimple-main",
 297             children=[
 298                 dbc.NavItem(
 299                     dbc.NavLink(
 300                         C.NEWS_TITLE,
 301                         disabled=True,
 302                         external_link=True,
 303                         href="#"
 304                     )
 305                 )
 306             ],
 307             brand=C.BRAND,
 308             brand_href="/",
 309             brand_external_link=True,
 310             class_name="p-2",
 311             fluid=True
 312         )
 313
 314     def _add_ctrl_col(self) -> dbc.Col:
 315         """Add column with control panel. It is placed on the left side.
 316
 317         :returns: Column with the control panel.
 318         :rtype: dbc.Col
 319         """
 320         return dbc.Col([
 321             html.Div(
 322                 children=self._add_ctrl_panel(),
 323                 className="sticky-top"
 324             )
 325         ])
 326
 327     def _add_plotting_col(self) -> dbc.Col:
 328         """Add column with tables. It is placed on the right side.
 329
 330         :returns: Column with tables.
 331         :rtype: dbc.Col
 332         """
 333         return dbc.Col(
 334             id="col-plotting-area",
 335             children=[
 336                 dbc.Spinner(
 337                     children=[
 338                         dbc.Row(
 339                             id="plotting-area",
 340                             class_name="g-0 p-0",
 341                             children=[
 342                                 C.PLACEHOLDER
 343                             ]
 344                         )
 345                     ]
 346                 )
 347             ],
 348             width=9
 349         )
 350
 351     def _add_ctrl_panel(self) -> list:
 352         """Add control panel.
 353
 354         :returns: Control panel.
 355         :rtype: list
 356         """
 357         return [
 358             dbc.Row(
 359                 class_name="g-0 p-1",
 360                 children=[
 361                     dbc.ButtonGroup(
 362                         id="bg-time-period",
 363                         children=[
 364                             dbc.Button(
 365                                 id="period-last",
 366                                 children="Last Run",
 367                                 className="me-1",
 368                                 outline=True,
 369                                 color="info"
 370                             ),
 371                             dbc.Button(
 372                                 id="period-short",
 373                                 children=f"Last {C.NEWS_SHORT} Runs",
 374                                 className="me-1",
 375                                 outline=True,
 376                                 active=True,
 377                                 color="info"
 378                             ),
 379                             dbc.Button(
 380                                 id="period-long",
 381                                 children="All Runs",
 382                                 className="me-1",
 383                                 outline=True,
 384                                 color="info"
 385                             )
 386                         ]
 387                     )
 388                 ]
 389             )
 390         ]
 391
 392     def _get_plotting_area(
 393             self,
 394             period: int,
 395             url: str
 396         ) -> list:
 397         """Generate the plotting area with all its content.
 398
 399         :param period: The time period for summary tables.
 400         :param url: URL to be displayed in the modal window.
 401         :type period: int
 402         :type url: str
 403         :returns: The content of the plotting area.
 404         :rtype: list
 405         """
 406         return [
 407             dbc.Row(
 408                 id="row-table",
 409                 class_name="g-0 p-1",
 410                 children=table_summary(self._data, self._jobs, period)
 411             ),
 412             dbc.Row(
 413                 [
 414                     dbc.Col([html.Div(
 415                         [
 416                             dbc.Button(
 417                                 id="plot-btn-url",
 418                                 children="Show URL",
 419                                 class_name="me-1",
 420                                 color="info",
 421                                 style={
 422                                     "text-transform": "none",
 423                                     "padding": "0rem 1rem"
 424                                 }
 425                             ),
 426                             dbc.Modal(
 427                                 [
 428                                     dbc.ModalHeader(dbc.ModalTitle("URL")),
 429                                     dbc.ModalBody(url)
 430                                 ],
 431                                 id="plot-mod-url",
 432                                 size="xl",
 433                                 is_open=False,
 434                                 scrollable=True
 435                             )
 436                         ],
 437                         className=\
 438                             "d-grid gap-0 d-md-flex justify-content-md-end"
 439                     )])
 440                 ],
 441                 class_name="g-0 p-0"
 442             )
 443         ]
 444
 445     def callbacks(self, app):
 446         """Callbacks for the whole application.
 447
 448         :param app: The application.
 449         :type app: Flask
 450         """
 451
 452         @app.callback(
 453             Output("plotting-area", "children"),
 454             Output("period-last", "active"),
 455             Output("period-short", "active"),
 456             Output("period-long", "active"),
 457             Input("url", "href"),
 458             Input("period-last", "n_clicks"),
 459             Input("period-short", "n_clicks"),
 460             Input("period-long", "n_clicks")
 461         )
 462         def _update_application(href: str, *_) -> tuple:
 463             """Update the application when the event is detected.
 464
 465             :returns: New values for web page elements.
 466             :rtype: tuple
 467             """
 468
 469             periods = {
 470                 "period-last": C.NEWS_LAST,
 471                 "period-short": C.NEWS_SHORT,
 472                 "period-long": C.NEWS_LONG
 473             }
 474             actives = {
 475                 "period-last": (True, False, False),
 476                 "period-short": (False, True, False),
 477                 "period-long": (False, False, True)
 478             }
 479
 480             # Parse the url:
 481             parsed_url = url_decode(href)
 482             if parsed_url:
 483                 url_params = parsed_url["params"]
 484             else:
 485                 url_params = None
 486
 487             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 488             if trigger_id == "url" and url_params:
 489                 trigger_id = url_params.get("period", list())[0]
 490
 491             ret_val = [
 492                 self._get_plotting_area(
 493                     periods.get(trigger_id, self._default_period),
 494                     gen_new_url(parsed_url, {"period": trigger_id})
 495                 )
 496             ]
 497             ret_val.extend(actives.get(trigger_id, self._default_active))
 498             return ret_val
 499
 500         @app.callback(
 501             Output("plot-mod-url", "is_open"),
 502             [Input("plot-btn-url", "n_clicks")],
 503             [State("plot-mod-url", "is_open")],
 504         )
 505         def toggle_plot_mod_url(n, is_open):
 506             """Toggle the modal window with url.
 507             """
 508             if n:
 509                 return not is_open
 510             return is_open