csit.infra.dash/app/cdash/news/layout.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import pandas as pd
  18 import dash_bootstrap_components as dbc
  19
  20 from flask import Flask
  21 from dash import dcc
  22 from dash import html
  23 from dash import callback_context
  24 from dash import Input, Output, State
  25
  26 from ..utils.constants import Constants as C
  27 from ..utils.utils import gen_new_url
  28 from ..utils.anomalies import classify_anomalies
  29 from ..utils.url_processing import url_decode
  30 from .tables import table_summary
  31
  32
  33 class Layout:
  34     """The layout of the dash app and the callbacks.
  35     """
  36
  37     def __init__(
  38             self,
  39             app: Flask,
  40             data_stats: pd.DataFrame,
  41             data_trending: pd.DataFrame,
  42             html_layout_file: str
  43         ) -> None:
  44         """Initialization:
  45         - save the input parameters,
  46         - read and pre-process the data,
  47         - prepare data for the control panel,
  48         - read HTML layout file,
  49         - read tooltips from the tooltip file.
  50
  51         :param app: Flask application running the dash application.
  52         :param data_stats: Pandas dataframe with staistical data.
  53         :param data_trending: Pandas dataframe with trending data.
  54         :param html_layout_file: Path and name of the file specifying the HTML
  55             layout of the dash application.
  56         :type app: Flask
  57         :type data_stats: pandas.DataFrame
  58         :type data_trending: pandas.DataFrame
  59         :type html_layout_file: str
  60         """
  61
  62         # Inputs
  63         self._app = app
  64         self._html_layout_file = html_layout_file
  65
  66         # Prepare information for the control panel:
  67         self._jobs = sorted(list(data_trending["job"].unique()))
  68         d_job_info = {
  69             "job": list(),
  70             "dut": list(),
  71             "ttype": list(),
  72             "cadence": list(),
  73             "tbed": list()
  74         }
  75         for job in self._jobs:
  76             lst_job = job.split("-")
  77             d_job_info["job"].append(job)
  78             d_job_info["dut"].append(lst_job[1])
  79             d_job_info["ttype"].append(lst_job[3])
  80             d_job_info["cadence"].append(lst_job[4])
  81             d_job_info["tbed"].append("-".join(lst_job[-2:]))
  82         self.job_info = pd.DataFrame.from_dict(d_job_info)
  83
  84         # Pre-process the data:
  85
  86         def _create_test_name(test: str) -> str:
  87             lst_tst = test.split(".")
  88             suite = lst_tst[-2].replace("2n1l-", "").replace("1n1l-", "").\
  89                 replace("2n-", "")
  90             return f"{suite.split('-')[0]}-{lst_tst[-1]}"
  91
  92         def _get_rindex(array: list, itm: any) -> int:
  93             return len(array) - 1 - array[::-1].index(itm)
  94
  95         tst_info = {
  96             "job": list(),
  97             "build": list(),
  98             "start": list(),
  99             "dut_type": list(),
 100             "dut_version": list(),
 101             "hosts": list(),
 102             "failed": list(),
 103             "regressions": list(),
 104             "progressions": list()
 105         }
 106         for job in self._jobs:
 107             # Create lists of failed tests:
 108             df_job = data_trending.loc[(data_trending["job"] == job)]
 109             last_build = str(max(pd.to_numeric(df_job["build"].unique())))
 110             df_build = df_job.loc[(df_job["build"] == last_build)]
 111             tst_info["job"].append(job)
 112             tst_info["build"].append(last_build)
 113             tst_info["start"].append(data_stats.loc[
 114                 (data_stats["job"] == job) &
 115                 (data_stats["build"] == last_build)
 116             ]["start_time"].iloc[-1].strftime('%Y-%m-%d %H:%M'))
 117             tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
 118             tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
 119             tst_info["hosts"].append(df_build["hosts"].iloc[-1])
 120             failed_tests = df_build.loc[(df_build["passed"] == False)]\
 121                 ["test_id"].to_list()
 122             l_failed = list()
 123             try:
 124                 for tst in failed_tests:
 125                     l_failed.append(_create_test_name(tst))
 126             except KeyError:
 127                 l_failed = list()
 128             tst_info["failed"].append(sorted(l_failed))
 129
 130             # Create lists of regressions and progressions:
 131             l_reg = list()
 132             l_prog = list()
 133
 134             tests = df_job["test_id"].unique()
 135             for test in tests:
 136                 tst_data = df_job.loc[(
 137                     (df_job["test_id"] == test) &
 138                     (df_job["passed"] == True)
 139                 )].sort_values(by="start_time", ignore_index=True)
 140                 if "-ndrpdr" in test:
 141                     tst_data = tst_data.dropna(
 142                         subset=["result_pdr_lower_rate_value", ]
 143                     )
 144                     if tst_data.empty:
 145                         continue
 146                     x_axis = tst_data["start_time"].tolist()
 147                     try:
 148                         anomalies, _, _ = classify_anomalies({
 149                             k: v for k, v in zip(
 150                                 x_axis,
 151                                 tst_data["result_ndr_lower_rate_value"].tolist()
 152                             )
 153                         })
 154                     except ValueError:
 155                         continue
 156                     if "progression" in anomalies:
 157                         l_prog.append((
 158                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 159                             x_axis[_get_rindex(anomalies, "progression")]
 160                         ))
 161                     if "regression" in anomalies:
 162                         l_reg.append((
 163                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 164                             x_axis[_get_rindex(anomalies, "regression")]
 165                         ))
 166                     try:
 167                         anomalies, _, _ = classify_anomalies({
 168                             k: v for k, v in zip(
 169                                 x_axis,
 170                                 tst_data["result_pdr_lower_rate_value"].tolist()
 171                             )
 172                         })
 173                     except ValueError:
 174                         continue
 175                     if "progression" in anomalies:
 176                         l_prog.append((
 177                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 178                             x_axis[_get_rindex(anomalies, "progression")]
 179                         ))
 180                     if "regression" in anomalies:
 181                         l_reg.append((
 182                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 183                             x_axis[_get_rindex(anomalies, "regression")]
 184                         ))
 185                 else:  # mrr
 186                     tst_data = tst_data.dropna(
 187                         subset=["result_receive_rate_rate_avg", ]
 188                     )
 189                     if tst_data.empty:
 190                         continue
 191                     x_axis = tst_data["start_time"].tolist()
 192                     try:
 193                         anomalies, _, _ = classify_anomalies({
 194                             k: v for k, v in zip(
 195                                 x_axis,
 196                                 tst_data["result_receive_rate_rate_avg"].\
 197                                     tolist()
 198                             )
 199                         })
 200                     except ValueError:
 201                         continue
 202                     if "progression" in anomalies:
 203                         l_prog.append((
 204                             _create_test_name(test),
 205                             x_axis[_get_rindex(anomalies, "progression")]
 206                         ))
 207                     if "regression" in anomalies:
 208                         l_reg.append((
 209                             _create_test_name(test),
 210                             x_axis[_get_rindex(anomalies, "regression")]
 211                         ))
 212
 213             tst_info["regressions"].append(
 214                 sorted(l_reg, key=lambda k: k[1], reverse=True))
 215             tst_info["progressions"].append(
 216                 sorted(l_prog, key=lambda k: k[1], reverse=True))
 217
 218         self._data = pd.DataFrame.from_dict(tst_info)
 219
 220         # Read from files:
 221         self._html_layout = str()
 222
 223         try:
 224             with open(self._html_layout_file, "r") as file_read:
 225                 self._html_layout = file_read.read()
 226         except IOError as err:
 227             raise RuntimeError(
 228                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 229             )
 230
 231         self._default_period = C.NEWS_SHORT
 232         self._default_active = (False, True, False)
 233
 234         # Callbacks:
 235         if self._app is not None and hasattr(self, 'callbacks'):
 236             self.callbacks(self._app)
 237
 238     @property
 239     def html_layout(self) -> dict:
 240         return self._html_layout
 241
 242     def add_content(self):
 243         """Top level method which generated the web page.
 244
 245         It generates:
 246         - Store for user input data,
 247         - Navigation bar,
 248         - Main area with control panel and ploting area.
 249
 250         If no HTML layout is provided, an error message is displayed instead.
 251
 252         :returns: The HTML div with the whole page.
 253         :rtype: html.Div
 254         """
 255
 256         if self.html_layout:
 257             return html.Div(
 258                 id="div-main",
 259                 className="small",
 260                 children=[
 261                     dcc.Location(id="url", refresh=False),
 262                     dbc.Row(
 263                         id="row-navbar",
 264                         class_name="g-0",
 265                         children=[
 266                             self._add_navbar()
 267                         ]
 268                     ),
 269                     dbc.Row(
 270                         id="row-main",
 271                         class_name="g-0",
 272                         children=[
 273                             self._add_ctrl_col(),
 274                             self._add_plotting_col()
 275                         ]
 276                     ),
 277                     dbc.Offcanvas(
 278                         class_name="w-75",
 279                         id="offcanvas-documentation",
 280                         title="Documentation",
 281                         placement="end",
 282                         is_open=False,
 283                         children=html.Iframe(
 284                             src=C.URL_DOC_TRENDING,
 285                             width="100%",
 286                             height="100%"
 287                         )
 288                     )
 289                 ]
 290             )
 291         else:
 292             return html.Div(
 293                 id="div-main-error",
 294                 children=[
 295                     dbc.Alert(
 296                         [
 297                             "An Error Occured"
 298                         ],
 299                         color="danger"
 300                     )
 301                 ]
 302             )
 303
 304     def _add_navbar(self):
 305         """Add nav element with navigation panel. It is placed on the top.
 306
 307         :returns: Navigation bar.
 308         :rtype: dbc.NavbarSimple
 309         """
 310
 311         return dbc.NavbarSimple(
 312             id="navbarsimple-main",
 313             children=[
 314                 dbc.NavItem(dbc.NavLink(
 315                     C.TREND_TITLE,
 316                     external_link=True,
 317                     href="/trending"
 318                 )),
 319                 dbc.NavItem(dbc.NavLink(
 320                     C.NEWS_TITLE,
 321                     active=True,
 322                     external_link=True,
 323                     href="/news"
 324                 )),
 325                 dbc.NavItem(dbc.NavLink(
 326                     C.STATS_TITLE,
 327                     external_link=True,
 328                     href="/stats"
 329                 )),
 330                 dbc.NavItem(dbc.NavLink(
 331                     "Documentation",
 332                     id="btn-documentation",
 333                 ))
 334             ],
 335             brand=C.BRAND,
 336             brand_href="/",
 337             brand_external_link=True,
 338             class_name="p-2",
 339             fluid=True
 340         )
 341
 342     def _add_ctrl_col(self) -> dbc.Col:
 343         """Add column with control panel. It is placed on the left side.
 344
 345         :returns: Column with the control panel.
 346         :rtype: dbc.Col
 347         """
 348         return dbc.Col([
 349             html.Div(
 350                 children=self._add_ctrl_panel(),
 351                 className="sticky-top"
 352             )
 353         ])
 354
 355     def _add_plotting_col(self) -> dbc.Col:
 356         """Add column with tables. It is placed on the right side.
 357
 358         :returns: Column with tables.
 359         :rtype: dbc.Col
 360         """
 361         return dbc.Col(
 362             id="col-plotting-area",
 363             children=[
 364                 dbc.Spinner(
 365                     children=[
 366                         dbc.Row(
 367                             id="plotting-area",
 368                             class_name="g-0 p-0",
 369                             children=[
 370                                 C.PLACEHOLDER
 371                             ]
 372                         )
 373                     ]
 374                 )
 375             ],
 376             width=9
 377         )
 378
 379     def _add_ctrl_panel(self) -> list:
 380         """Add control panel.
 381
 382         :returns: Control panel.
 383         :rtype: list
 384         """
 385         return [
 386             dbc.Row(
 387                 class_name="g-0 p-1",
 388                 children=[
 389                     dbc.ButtonGroup(
 390                         id="bg-time-period",
 391                         children=[
 392                             dbc.Button(
 393                                 id="period-last",
 394                                 children="Last Run",
 395                                 className="me-1",
 396                                 outline=True,
 397                                 color="info"
 398                             ),
 399                             dbc.Button(
 400                                 id="period-short",
 401                                 children=f"Last {C.NEWS_SHORT} Runs",
 402                                 className="me-1",
 403                                 outline=True,
 404                                 active=True,
 405                                 color="info"
 406                             ),
 407                             dbc.Button(
 408                                 id="period-long",
 409                                 children="All Runs",
 410                                 className="me-1",
 411                                 outline=True,
 412                                 color="info"
 413                             )
 414                         ]
 415                     )
 416                 ]
 417             )
 418         ]
 419
 420     def _get_plotting_area(
 421             self,
 422             period: int,
 423             url: str
 424         ) -> list:
 425         """Generate the plotting area with all its content.
 426
 427         :param period: The time period for summary tables.
 428         :param url: URL to be displayed in the modal window.
 429         :type period: int
 430         :type url: str
 431         :returns: The content of the plotting area.
 432         :rtype: list
 433         """
 434         return [
 435             dbc.Row(
 436                 id="row-table",
 437                 class_name="g-0 p-1",
 438                 children=table_summary(self._data, self._jobs, period)
 439             ),
 440             dbc.Row(
 441                 [
 442                     dbc.Col([html.Div(
 443                         [
 444                             dbc.Button(
 445                                 id="plot-btn-url",
 446                                 children="Show URL",
 447                                 class_name="me-1",
 448                                 color="info",
 449                                 style={
 450                                     "text-transform": "none",
 451                                     "padding": "0rem 1rem"
 452                                 }
 453                             ),
 454                             dbc.Modal(
 455                                 [
 456                                     dbc.ModalHeader(dbc.ModalTitle("URL")),
 457                                     dbc.ModalBody(url)
 458                                 ],
 459                                 id="plot-mod-url",
 460                                 size="xl",
 461                                 is_open=False,
 462                                 scrollable=True
 463                             )
 464                         ],
 465                         className=\
 466                             "d-grid gap-0 d-md-flex justify-content-md-end"
 467                     )])
 468                 ],
 469                 class_name="g-0 p-0"
 470             )
 471         ]
 472
 473     def callbacks(self, app):
 474         """Callbacks for the whole application.
 475
 476         :param app: The application.
 477         :type app: Flask
 478         """
 479
 480         @app.callback(
 481             Output("plotting-area", "children"),
 482             Output("period-last", "active"),
 483             Output("period-short", "active"),
 484             Output("period-long", "active"),
 485             Input("url", "href"),
 486             Input("period-last", "n_clicks"),
 487             Input("period-short", "n_clicks"),
 488             Input("period-long", "n_clicks")
 489         )
 490         def _update_application(href: str, *_) -> tuple:
 491             """Update the application when the event is detected.
 492
 493             :returns: New values for web page elements.
 494             :rtype: tuple
 495             """
 496
 497             periods = {
 498                 "period-last": C.NEWS_LAST,
 499                 "period-short": C.NEWS_SHORT,
 500                 "period-long": C.NEWS_LONG
 501             }
 502             actives = {
 503                 "period-last": (True, False, False),
 504                 "period-short": (False, True, False),
 505                 "period-long": (False, False, True)
 506             }
 507
 508             # Parse the url:
 509             parsed_url = url_decode(href)
 510             if parsed_url:
 511                 url_params = parsed_url["params"]
 512             else:
 513                 url_params = None
 514
 515             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 516             if trigger_id == "url" and url_params:
 517                 trigger_id = url_params.get("period", list())[0]
 518
 519             ret_val = [
 520                 self._get_plotting_area(
 521                     periods.get(trigger_id, self._default_period),
 522                     gen_new_url(parsed_url, {"period": trigger_id})
 523                 )
 524             ]
 525             ret_val.extend(actives.get(trigger_id, self._default_active))
 526             return ret_val
 527
 528         @app.callback(
 529             Output("plot-mod-url", "is_open"),
 530             [Input("plot-btn-url", "n_clicks")],
 531             [State("plot-mod-url", "is_open")],
 532         )
 533         def toggle_plot_mod_url(n, is_open):
 534             """Toggle the modal window with url.
 535             """
 536             if n:
 537                 return not is_open
 538             return is_open
 539
 540         @app.callback(
 541             Output("offcanvas-documentation", "is_open"),
 542             Input("btn-documentation", "n_clicks"),
 543             State("offcanvas-documentation", "is_open")
 544         )
 545         def toggle_offcanvas_documentation(n_clicks, is_open):
 546             if n_clicks:
 547                 return not is_open
 548             return is_open