csit.infra.dash/app/cdash/news/layout.py

   1 # Copyright (c) 2024 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import pandas as pd
  18 import dash_bootstrap_components as dbc
  19
  20 from flask import Flask
  21 from dash import dcc
  22 from dash import html
  23 from dash import callback_context
  24 from dash import Input, Output, State
  25
  26 from ..utils.constants import Constants as C
  27 from ..utils.utils import gen_new_url, navbar_trending
  28 from ..utils.anomalies import classify_anomalies
  29 from ..utils.url_processing import url_decode
  30 from .tables import table_summary
  31
  32
  33 class Layout:
  34     """The layout of the dash app and the callbacks.
  35     """
  36
  37     def __init__(
  38             self,
  39             app: Flask,
  40             data_stats: pd.DataFrame,
  41             data_trending: pd.DataFrame,
  42             html_layout_file: str
  43         ) -> None:
  44         """Initialization:
  45         - save the input parameters,
  46         - read and pre-process the data,
  47         - prepare data for the control panel,
  48         - read HTML layout file,
  49         - read tooltips from the tooltip file.
  50
  51         :param app: Flask application running the dash application.
  52         :param data_stats: Pandas dataframe with staistical data.
  53         :param data_trending: Pandas dataframe with trending data.
  54         :param html_layout_file: Path and name of the file specifying the HTML
  55             layout of the dash application.
  56         :type app: Flask
  57         :type data_stats: pandas.DataFrame
  58         :type data_trending: pandas.DataFrame
  59         :type html_layout_file: str
  60         """
  61
  62         # Inputs
  63         self._app = app
  64         self._html_layout_file = html_layout_file
  65
  66         # Prepare information for the control panel:
  67         self._jobs = sorted(list(data_trending["job"].unique()))
  68         d_job_info = {
  69             "job": list(),
  70             "dut": list(),
  71             "ttype": list(),
  72             "cadence": list(),
  73             "tbed": list()
  74         }
  75         for job in self._jobs:
  76             lst_job = job.split("-")
  77             d_job_info["job"].append(job)
  78             d_job_info["dut"].append(lst_job[1])
  79             d_job_info["ttype"].append(lst_job[3])
  80             d_job_info["cadence"].append(lst_job[4])
  81             d_job_info["tbed"].append("-".join(lst_job[-2:]))
  82         self.job_info = pd.DataFrame.from_dict(d_job_info)
  83
  84         # Pre-process the data:
  85
  86         def _create_test_name(test: str) -> str:
  87             lst_tst = test.split(".")
  88             suite = lst_tst[-2].replace("2n1l-", "").replace("1n1l-", "").\
  89                 replace("2n-", "")
  90             return f"{suite.split('-')[0]}-{lst_tst[-1]}"
  91
  92         def _get_rindex(array: list, itm: any) -> int:
  93             return len(array) - 1 - array[::-1].index(itm)
  94
  95         tst_info = {
  96             "job": list(),
  97             "build": list(),
  98             "start": list(),
  99             "dut_type": list(),
 100             "dut_version": list(),
 101             "hosts": list(),
 102             "failed": list(),
 103             "regressions": list(),
 104             "progressions": list()
 105         }
 106         for job in self._jobs:
 107             # Create lists of failed tests:
 108             df_job = data_trending.loc[(data_trending["job"] == job)]
 109             last_build = str(max(pd.to_numeric(df_job["build"].unique())))
 110             df_build = df_job.loc[(df_job["build"] == last_build)]
 111             tst_info["job"].append(job)
 112             tst_info["build"].append(last_build)
 113             tst_info["start"].append(data_stats.loc[
 114                 (data_stats["job"] == job) &
 115                 (data_stats["build"] == last_build)
 116             ]["start_time"].iloc[-1].strftime('%Y-%m-%d %H:%M'))
 117             tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
 118             tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
 119             tst_info["hosts"].append(df_build["hosts"].iloc[-1])
 120             failed_tests = df_build.loc[(df_build["passed"] == False)]\
 121                 ["test_id"].to_list()
 122             l_failed = list()
 123             try:
 124                 for tst in failed_tests:
 125                     l_failed.append(_create_test_name(tst))
 126             except KeyError:
 127                 l_failed = list()
 128             tst_info["failed"].append(sorted(l_failed))
 129
 130             # Create lists of regressions and progressions:
 131             l_reg = list()
 132             l_prog = list()
 133
 134             tests = df_job["test_id"].unique()
 135             for test in tests:
 136                 tst_data = df_job.loc[(
 137                     (df_job["test_id"] == test) &
 138                     (df_job["passed"] == True)
 139                 )].sort_values(by="start_time", ignore_index=True)
 140                 if "-ndrpdr" in test:
 141                     tst_data = tst_data.dropna(
 142                         subset=["result_pdr_lower_rate_value", ]
 143                     )
 144                     if tst_data.empty:
 145                         continue
 146                     x_axis = tst_data["start_time"].tolist()
 147                     try:
 148                         anomalies, _, _ = classify_anomalies({
 149                             k: v for k, v in zip(
 150                                 x_axis,
 151                                 tst_data["result_ndr_lower_rate_value"].tolist()
 152                             )
 153                         })
 154                     except ValueError:
 155                         continue
 156                     if "progression" in anomalies:
 157                         l_prog.append((
 158                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 159                             x_axis[_get_rindex(anomalies, "progression")]
 160                         ))
 161                     if "regression" in anomalies:
 162                         l_reg.append((
 163                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 164                             x_axis[_get_rindex(anomalies, "regression")]
 165                         ))
 166                     try:
 167                         anomalies, _, _ = classify_anomalies({
 168                             k: v for k, v in zip(
 169                                 x_axis,
 170                                 tst_data["result_pdr_lower_rate_value"].tolist()
 171                             )
 172                         })
 173                     except ValueError:
 174                         continue
 175                     if "progression" in anomalies:
 176                         l_prog.append((
 177                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 178                             x_axis[_get_rindex(anomalies, "progression")]
 179                         ))
 180                     if "regression" in anomalies:
 181                         l_reg.append((
 182                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 183                             x_axis[_get_rindex(anomalies, "regression")]
 184                         ))
 185                 else:  # mrr
 186                     tst_data = tst_data.dropna(
 187                         subset=["result_receive_rate_rate_avg", ]
 188                     )
 189                     if tst_data.empty:
 190                         continue
 191                     x_axis = tst_data["start_time"].tolist()
 192                     try:
 193                         anomalies, _, _ = classify_anomalies({
 194                             k: v for k, v in zip(
 195                                 x_axis,
 196                                 tst_data["result_receive_rate_rate_avg"].\
 197                                     tolist()
 198                             )
 199                         })
 200                     except ValueError:
 201                         continue
 202                     if "progression" in anomalies:
 203                         l_prog.append((
 204                             _create_test_name(test),
 205                             x_axis[_get_rindex(anomalies, "progression")]
 206                         ))
 207                     if "regression" in anomalies:
 208                         l_reg.append((
 209                             _create_test_name(test),
 210                             x_axis[_get_rindex(anomalies, "regression")]
 211                         ))
 212
 213             tst_info["regressions"].append(
 214                 sorted(l_reg, key=lambda k: k[1], reverse=True))
 215             tst_info["progressions"].append(
 216                 sorted(l_prog, key=lambda k: k[1], reverse=True))
 217
 218         self._data = pd.DataFrame.from_dict(tst_info)
 219
 220         # Read from files:
 221         self._html_layout = str()
 222
 223         try:
 224             with open(self._html_layout_file, "r") as file_read:
 225                 self._html_layout = file_read.read()
 226         except IOError as err:
 227             raise RuntimeError(
 228                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 229             )
 230
 231         self._default_period = C.NEWS_SHORT
 232         self._default_active = (False, True, False)
 233
 234         # Callbacks:
 235         if self._app is not None and hasattr(self, 'callbacks'):
 236             self.callbacks(self._app)
 237
 238     @property
 239     def html_layout(self) -> dict:
 240         return self._html_layout
 241
 242     def add_content(self):
 243         """Top level method which generated the web page.
 244
 245         It generates:
 246         - Store for user input data,
 247         - Navigation bar,
 248         - Main area with control panel and ploting area.
 249
 250         If no HTML layout is provided, an error message is displayed instead.
 251
 252         :returns: The HTML div with the whole page.
 253         :rtype: html.Div
 254         """
 255
 256         if self.html_layout:
 257             return html.Div(
 258                 id="div-main",
 259                 className="small",
 260                 children=[
 261                     dcc.Location(id="url", refresh=False),
 262                     dbc.Row(
 263                         id="row-navbar",
 264                         class_name="g-0",
 265                         children=[navbar_trending((False, True, False, False))]
 266                     ),
 267                     dbc.Row(
 268                         id="row-main",
 269                         class_name="g-0",
 270                         children=[
 271                             self._add_ctrl_col(),
 272                             self._add_plotting_col()
 273                         ]
 274                     ),
 275                     dbc.Offcanvas(
 276                         class_name="w-75",
 277                         id="offcanvas-documentation",
 278                         title="Documentation",
 279                         placement="end",
 280                         is_open=False,
 281                         children=html.Iframe(
 282                             src=C.URL_DOC_TRENDING,
 283                             width="100%",
 284                             height="100%"
 285                         )
 286                     )
 287                 ]
 288             )
 289         else:
 290             return html.Div(
 291                 id="div-main-error",
 292                 children=[
 293                     dbc.Alert(
 294                         [
 295                             "An Error Occured"
 296                         ],
 297                         color="danger"
 298                     )
 299                 ]
 300             )
 301
 302     def _add_ctrl_col(self) -> dbc.Col:
 303         """Add column with control panel. It is placed on the left side.
 304
 305         :returns: Column with the control panel.
 306         :rtype: dbc.Col
 307         """
 308         return dbc.Col([
 309             html.Div(
 310                 children=self._add_ctrl_panel(),
 311                 className="sticky-top"
 312             )
 313         ])
 314
 315     def _add_plotting_col(self) -> dbc.Col:
 316         """Add column with tables. It is placed on the right side.
 317
 318         :returns: Column with tables.
 319         :rtype: dbc.Col
 320         """
 321         return dbc.Col(
 322             id="col-plotting-area",
 323             children=[
 324                 dbc.Spinner(
 325                     children=[
 326                         dbc.Row(
 327                             id="plotting-area",
 328                             class_name="g-0 p-0",
 329                             children=[
 330                                 C.PLACEHOLDER
 331                             ]
 332                         )
 333                     ]
 334                 )
 335             ],
 336             width=9
 337         )
 338
 339     def _add_ctrl_panel(self) -> list:
 340         """Add control panel.
 341
 342         :returns: Control panel.
 343         :rtype: list
 344         """
 345         return [
 346             dbc.Row(
 347                 class_name="g-0 p-1",
 348                 children=[
 349                     dbc.ButtonGroup(
 350                         id="bg-time-period",
 351                         children=[
 352                             dbc.Button(
 353                                 id="period-last",
 354                                 children="Last Run",
 355                                 className="me-1",
 356                                 outline=True,
 357                                 color="info"
 358                             ),
 359                             dbc.Button(
 360                                 id="period-short",
 361                                 children=f"Last {C.NEWS_SHORT} Runs",
 362                                 className="me-1",
 363                                 outline=True,
 364                                 active=True,
 365                                 color="info"
 366                             ),
 367                             dbc.Button(
 368                                 id="period-long",
 369                                 children="All Runs",
 370                                 className="me-1",
 371                                 outline=True,
 372                                 color="info"
 373                             )
 374                         ]
 375                     )
 376                 ]
 377             )
 378         ]
 379
 380     def _get_plotting_area(
 381             self,
 382             period: int,
 383             url: str
 384         ) -> list:
 385         """Generate the plotting area with all its content.
 386
 387         :param period: The time period for summary tables.
 388         :param url: URL to be displayed in the modal window.
 389         :type period: int
 390         :type url: str
 391         :returns: The content of the plotting area.
 392         :rtype: list
 393         """
 394         return [
 395             dbc.Row(
 396                 id="row-table",
 397                 class_name="g-0 p-1",
 398                 children=table_summary(self._data, self._jobs, period)
 399             ),
 400             dbc.Row(
 401                 [
 402                     dbc.Col([html.Div(
 403                         [
 404                             dbc.Button(
 405                                 id="plot-btn-url",
 406                                 children="Show URL",
 407                                 class_name="me-1",
 408                                 color="info",
 409                                 style={
 410                                     "text-transform": "none",
 411                                     "padding": "0rem 1rem"
 412                                 }
 413                             ),
 414                             dbc.Modal(
 415                                 [
 416                                     dbc.ModalHeader(dbc.ModalTitle("URL")),
 417                                     dbc.ModalBody(url)
 418                                 ],
 419                                 id="plot-mod-url",
 420                                 size="xl",
 421                                 is_open=False,
 422                                 scrollable=True
 423                             )
 424                         ],
 425                         className=\
 426                             "d-grid gap-0 d-md-flex justify-content-md-end"
 427                     )])
 428                 ],
 429                 class_name="g-0 p-0"
 430             )
 431         ]
 432
 433     def callbacks(self, app):
 434         """Callbacks for the whole application.
 435
 436         :param app: The application.
 437         :type app: Flask
 438         """
 439
 440         @app.callback(
 441             Output("plotting-area", "children"),
 442             Output("period-last", "active"),
 443             Output("period-short", "active"),
 444             Output("period-long", "active"),
 445             Input("url", "href"),
 446             Input("period-last", "n_clicks"),
 447             Input("period-short", "n_clicks"),
 448             Input("period-long", "n_clicks")
 449         )
 450         def _update_application(href: str, *_) -> tuple:
 451             """Update the application when the event is detected.
 452
 453             :returns: New values for web page elements.
 454             :rtype: tuple
 455             """
 456
 457             periods = {
 458                 "period-last": C.NEWS_LAST,
 459                 "period-short": C.NEWS_SHORT,
 460                 "period-long": C.NEWS_LONG
 461             }
 462             actives = {
 463                 "period-last": (True, False, False),
 464                 "period-short": (False, True, False),
 465                 "period-long": (False, False, True)
 466             }
 467
 468             # Parse the url:
 469             parsed_url = url_decode(href)
 470             if parsed_url:
 471                 url_params = parsed_url["params"]
 472             else:
 473                 url_params = None
 474
 475             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 476             if trigger_id == "url" and url_params:
 477                 trigger_id = url_params.get("period", list())[0]
 478
 479             ret_val = [
 480                 self._get_plotting_area(
 481                     periods.get(trigger_id, self._default_period),
 482                     gen_new_url(parsed_url, {"period": trigger_id})
 483                 )
 484             ]
 485             ret_val.extend(actives.get(trigger_id, self._default_active))
 486             return ret_val
 487
 488         @app.callback(
 489             Output("plot-mod-url", "is_open"),
 490             Input("plot-btn-url", "n_clicks"),
 491             State("plot-mod-url", "is_open")
 492         )
 493         def toggle_plot_mod_url(n, is_open):
 494             """Toggle the modal window with url.
 495             """
 496             if n:
 497                 return not is_open
 498             return is_open
 499
 500         @app.callback(
 501             Output("offcanvas-documentation", "is_open"),
 502             Input("btn-documentation", "n_clicks"),
 503             State("offcanvas-documentation", "is_open")
 504         )
 505         def toggle_offcanvas_documentation(n_clicks, is_open):
 506             if n_clicks:
 507                 return not is_open
 508             return is_open