csit.infra.dash/app/cdash/news/layout.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import pandas as pd
  18 import dash_bootstrap_components as dbc
  19
  20 from flask import Flask
  21 from dash import dcc
  22 from dash import html
  23 from dash import callback_context
  24 from dash import Input, Output, State
  25
  26 from ..utils.constants import Constants as C
  27 from ..utils.utils import gen_new_url
  28 from ..utils.anomalies import classify_anomalies
  29 from ..utils.url_processing import url_decode
  30 from .tables import table_summary
  31
  32
  33 class Layout:
  34     """The layout of the dash app and the callbacks.
  35     """
  36
  37     def __init__(
  38             self,
  39             app: Flask,
  40             data_stats: pd.DataFrame,
  41             data_trending: pd.DataFrame,
  42             html_layout_file: str
  43         ) -> None:
  44         """Initialization:
  45         - save the input parameters,
  46         - read and pre-process the data,
  47         - prepare data for the control panel,
  48         - read HTML layout file,
  49         - read tooltips from the tooltip file.
  50
  51         :param app: Flask application running the dash application.
  52         :param data_stats: Pandas dataframe with staistical data.
  53         :param data_trending: Pandas dataframe with trending data.
  54         :param html_layout_file: Path and name of the file specifying the HTML
  55             layout of the dash application.
  56         :type app: Flask
  57         :type data_stats: pandas.DataFrame
  58         :type data_trending: pandas.DataFrame
  59         :type html_layout_file: str
  60         """
  61
  62         # Inputs
  63         self._app = app
  64         self._html_layout_file = html_layout_file
  65
  66         # Prepare information for the control panel:
  67         self._jobs = sorted(list(data_trending["job"].unique()))
  68         d_job_info = {
  69             "job": list(),
  70             "dut": list(),
  71             "ttype": list(),
  72             "cadence": list(),
  73             "tbed": list()
  74         }
  75         for job in self._jobs:
  76             lst_job = job.split("-")
  77             d_job_info["job"].append(job)
  78             d_job_info["dut"].append(lst_job[1])
  79             d_job_info["ttype"].append(lst_job[3])
  80             d_job_info["cadence"].append(lst_job[4])
  81             d_job_info["tbed"].append("-".join(lst_job[-2:]))
  82         self.job_info = pd.DataFrame.from_dict(d_job_info)
  83
  84         # Pre-process the data:
  85
  86         def _create_test_name(test: str) -> str:
  87             lst_tst = test.split(".")
  88             suite = lst_tst[-2].replace("2n1l-", "").replace("1n1l-", "").\
  89                 replace("2n-", "")
  90             return f"{suite.split('-')[0]}-{lst_tst[-1]}"
  91
  92         def _get_rindex(array: list, itm: any) -> int:
  93             return len(array) - 1 - array[::-1].index(itm)
  94
  95         tst_info = {
  96             "job": list(),
  97             "build": list(),
  98             "start": list(),
  99             "dut_type": list(),
 100             "dut_version": list(),
 101             "hosts": list(),
 102             "failed": list(),
 103             "regressions": list(),
 104             "progressions": list()
 105         }
 106         for job in self._jobs:
 107             # Create lists of failed tests:
 108             df_job = data_trending.loc[(data_trending["job"] == job)]
 109             last_build = str(max(pd.to_numeric(df_job["build"].unique())))
 110             df_build = df_job.loc[(df_job["build"] == last_build)]
 111             tst_info["job"].append(job)
 112             tst_info["build"].append(last_build)
 113             tst_info["start"].append(data_stats.loc[
 114                 (data_stats["job"] == job) &
 115                 (data_stats["build"] == last_build)
 116             ]["start_time"].iloc[-1].strftime('%Y-%m-%d %H:%M'))
 117             tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
 118             tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
 119             tst_info["hosts"].append(df_build["hosts"].iloc[-1])
 120             failed_tests = df_build.loc[(df_build["passed"] == False)]\
 121                 ["test_id"].to_list()
 122             l_failed = list()
 123             try:
 124                 for tst in failed_tests:
 125                     l_failed.append(_create_test_name(tst))
 126             except KeyError:
 127                 l_failed = list()
 128             tst_info["failed"].append(sorted(l_failed))
 129
 130             # Create lists of regressions and progressions:
 131             l_reg = list()
 132             l_prog = list()
 133
 134             tests = df_job["test_id"].unique()
 135             for test in tests:
 136                 tst_data = df_job.loc[(
 137                     (df_job["test_id"] == test) &
 138                     (df_job["passed"] == True)
 139                 )].sort_values(by="start_time", ignore_index=True)
 140                 if "-ndrpdr" in test:
 141                     tst_data = tst_data.dropna(
 142                         subset=["result_pdr_lower_rate_value", ]
 143                     )
 144                     if tst_data.empty:
 145                         continue
 146                     x_axis = tst_data["start_time"].tolist()
 147                     try:
 148                         anomalies, _, _ = classify_anomalies({
 149                             k: v for k, v in zip(
 150                                 x_axis,
 151                                 tst_data["result_ndr_lower_rate_value"].tolist()
 152                             )
 153                         })
 154                     except ValueError:
 155                         continue
 156                     if "progression" in anomalies:
 157                         l_prog.append((
 158                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 159                             x_axis[_get_rindex(anomalies, "progression")]
 160                         ))
 161                     if "regression" in anomalies:
 162                         l_reg.append((
 163                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 164                             x_axis[_get_rindex(anomalies, "regression")]
 165                         ))
 166                     try:
 167                         anomalies, _, _ = classify_anomalies({
 168                             k: v for k, v in zip(
 169                                 x_axis,
 170                                 tst_data["result_pdr_lower_rate_value"].tolist()
 171                             )
 172                         })
 173                     except ValueError:
 174                         continue
 175                     if "progression" in anomalies:
 176                         l_prog.append((
 177                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 178                             x_axis[_get_rindex(anomalies, "progression")]
 179                         ))
 180                     if "regression" in anomalies:
 181                         l_reg.append((
 182                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 183                             x_axis[_get_rindex(anomalies, "regression")]
 184                         ))
 185                 else:  # mrr
 186                     tst_data = tst_data.dropna(
 187                         subset=["result_receive_rate_rate_avg", ]
 188                     )
 189                     if tst_data.empty:
 190                         continue
 191                     x_axis = tst_data["start_time"].tolist()
 192                     try:
 193                         anomalies, _, _ = classify_anomalies({
 194                             k: v for k, v in zip(
 195                                 x_axis,
 196                                 tst_data["result_receive_rate_rate_avg"].\
 197                                     tolist()
 198                             )
 199                         })
 200                     except ValueError:
 201                         continue
 202                     if "progression" in anomalies:
 203                         l_prog.append((
 204                             _create_test_name(test),
 205                             x_axis[_get_rindex(anomalies, "progression")]
 206                         ))
 207                     if "regression" in anomalies:
 208                         l_reg.append((
 209                             _create_test_name(test),
 210                             x_axis[_get_rindex(anomalies, "regression")]
 211                         ))
 212
 213             tst_info["regressions"].append(
 214                 sorted(l_reg, key=lambda k: k[1], reverse=True))
 215             tst_info["progressions"].append(
 216                 sorted(l_prog, key=lambda k: k[1], reverse=True))
 217
 218         self._data = pd.DataFrame.from_dict(tst_info)
 219
 220         # Read from files:
 221         self._html_layout = str()
 222
 223         try:
 224             with open(self._html_layout_file, "r") as file_read:
 225                 self._html_layout = file_read.read()
 226         except IOError as err:
 227             raise RuntimeError(
 228                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 229             )
 230
 231         self._default_period = C.NEWS_SHORT
 232         self._default_active = (False, True, False)
 233
 234         # Callbacks:
 235         if self._app is not None and hasattr(self, 'callbacks'):
 236             self.callbacks(self._app)
 237
 238     @property
 239     def html_layout(self) -> dict:
 240         return self._html_layout
 241
 242     def add_content(self):
 243         """Top level method which generated the web page.
 244
 245         It generates:
 246         - Store for user input data,
 247         - Navigation bar,
 248         - Main area with control panel and ploting area.
 249
 250         If no HTML layout is provided, an error message is displayed instead.
 251
 252         :returns: The HTML div with the whole page.
 253         :rtype: html.Div
 254         """
 255
 256         if self.html_layout:
 257             return html.Div(
 258                 id="div-main",
 259                 className="small",
 260                 children=[
 261                     dcc.Location(id="url", refresh=False),
 262                     dbc.Row(
 263                         id="row-navbar",
 264                         class_name="g-0",
 265                         children=[
 266                             self._add_navbar()
 267                         ]
 268                     ),
 269                     dbc.Row(
 270                         id="row-main",
 271                         class_name="g-0",
 272                         children=[
 273                             self._add_ctrl_col(),
 274                             self._add_plotting_col()
 275                         ]
 276                     )
 277                 ]
 278             )
 279         else:
 280             return html.Div(
 281                 id="div-main-error",
 282                 children=[
 283                     dbc.Alert(
 284                         [
 285                             "An Error Occured"
 286                         ],
 287                         color="danger"
 288                     )
 289                 ]
 290             )
 291
 292     def _add_navbar(self):
 293         """Add nav element with navigation panel. It is placed on the top.
 294
 295         :returns: Navigation bar.
 296         :rtype: dbc.NavbarSimple
 297         """
 298
 299         return dbc.NavbarSimple(
 300             id="navbarsimple-main",
 301             children=[
 302                 dbc.NavItem(
 303                     dbc.NavLink(
 304                         C.NEWS_TITLE,
 305                         disabled=True,
 306                         external_link=True,
 307                         href="#"
 308                     )
 309                 )
 310             ],
 311             brand=C.BRAND,
 312             brand_href="/",
 313             brand_external_link=True,
 314             class_name="p-2",
 315             fluid=True
 316         )
 317
 318     def _add_ctrl_col(self) -> dbc.Col:
 319         """Add column with control panel. It is placed on the left side.
 320
 321         :returns: Column with the control panel.
 322         :rtype: dbc.Col
 323         """
 324         return dbc.Col([
 325             html.Div(
 326                 children=self._add_ctrl_panel(),
 327                 className="sticky-top"
 328             )
 329         ])
 330
 331     def _add_plotting_col(self) -> dbc.Col:
 332         """Add column with tables. It is placed on the right side.
 333
 334         :returns: Column with tables.
 335         :rtype: dbc.Col
 336         """
 337         return dbc.Col(
 338             id="col-plotting-area",
 339             children=[
 340                 dbc.Spinner(
 341                     children=[
 342                         dbc.Row(
 343                             id="plotting-area",
 344                             class_name="g-0 p-0",
 345                             children=[
 346                                 C.PLACEHOLDER
 347                             ]
 348                         )
 349                     ]
 350                 )
 351             ],
 352             width=9
 353         )
 354
 355     def _add_ctrl_panel(self) -> list:
 356         """Add control panel.
 357
 358         :returns: Control panel.
 359         :rtype: list
 360         """
 361         return [
 362             dbc.Row(
 363                 class_name="g-0 p-1",
 364                 children=[
 365                     dbc.ButtonGroup(
 366                         id="bg-time-period",
 367                         children=[
 368                             dbc.Button(
 369                                 id="period-last",
 370                                 children="Last Run",
 371                                 className="me-1",
 372                                 outline=True,
 373                                 color="info"
 374                             ),
 375                             dbc.Button(
 376                                 id="period-short",
 377                                 children=f"Last {C.NEWS_SHORT} Runs",
 378                                 className="me-1",
 379                                 outline=True,
 380                                 active=True,
 381                                 color="info"
 382                             ),
 383                             dbc.Button(
 384                                 id="period-long",
 385                                 children="All Runs",
 386                                 className="me-1",
 387                                 outline=True,
 388                                 color="info"
 389                             )
 390                         ]
 391                     )
 392                 ]
 393             )
 394         ]
 395
 396     def _get_plotting_area(
 397             self,
 398             period: int,
 399             url: str
 400         ) -> list:
 401         """Generate the plotting area with all its content.
 402
 403         :param period: The time period for summary tables.
 404         :param url: URL to be displayed in the modal window.
 405         :type period: int
 406         :type url: str
 407         :returns: The content of the plotting area.
 408         :rtype: list
 409         """
 410         return [
 411             dbc.Row(
 412                 id="row-table",
 413                 class_name="g-0 p-1",
 414                 children=table_summary(self._data, self._jobs, period)
 415             ),
 416             dbc.Row(
 417                 [
 418                     dbc.Col([html.Div(
 419                         [
 420                             dbc.Button(
 421                                 id="plot-btn-url",
 422                                 children="Show URL",
 423                                 class_name="me-1",
 424                                 color="info",
 425                                 style={
 426                                     "text-transform": "none",
 427                                     "padding": "0rem 1rem"
 428                                 }
 429                             ),
 430                             dbc.Modal(
 431                                 [
 432                                     dbc.ModalHeader(dbc.ModalTitle("URL")),
 433                                     dbc.ModalBody(url)
 434                                 ],
 435                                 id="plot-mod-url",
 436                                 size="xl",
 437                                 is_open=False,
 438                                 scrollable=True
 439                             )
 440                         ],
 441                         className=\
 442                             "d-grid gap-0 d-md-flex justify-content-md-end"
 443                     )])
 444                 ],
 445                 class_name="g-0 p-0"
 446             )
 447         ]
 448
 449     def callbacks(self, app):
 450         """Callbacks for the whole application.
 451
 452         :param app: The application.
 453         :type app: Flask
 454         """
 455
 456         @app.callback(
 457             Output("plotting-area", "children"),
 458             Output("period-last", "active"),
 459             Output("period-short", "active"),
 460             Output("period-long", "active"),
 461             Input("url", "href"),
 462             Input("period-last", "n_clicks"),
 463             Input("period-short", "n_clicks"),
 464             Input("period-long", "n_clicks")
 465         )
 466         def _update_application(href: str, *_) -> tuple:
 467             """Update the application when the event is detected.
 468
 469             :returns: New values for web page elements.
 470             :rtype: tuple
 471             """
 472
 473             periods = {
 474                 "period-last": C.NEWS_LAST,
 475                 "period-short": C.NEWS_SHORT,
 476                 "period-long": C.NEWS_LONG
 477             }
 478             actives = {
 479                 "period-last": (True, False, False),
 480                 "period-short": (False, True, False),
 481                 "period-long": (False, False, True)
 482             }
 483
 484             # Parse the url:
 485             parsed_url = url_decode(href)
 486             if parsed_url:
 487                 url_params = parsed_url["params"]
 488             else:
 489                 url_params = None
 490
 491             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 492             if trigger_id == "url" and url_params:
 493                 trigger_id = url_params.get("period", list())[0]
 494
 495             ret_val = [
 496                 self._get_plotting_area(
 497                     periods.get(trigger_id, self._default_period),
 498                     gen_new_url(parsed_url, {"period": trigger_id})
 499                 )
 500             ]
 501             ret_val.extend(actives.get(trigger_id, self._default_active))
 502             return ret_val
 503
 504         @app.callback(
 505             Output("plot-mod-url", "is_open"),
 506             [Input("plot-btn-url", "n_clicks")],
 507             [State("plot-mod-url", "is_open")],
 508         )
 509         def toggle_plot_mod_url(n, is_open):
 510             """Toggle the modal window with url.
 511             """
 512             if n:
 513                 return not is_open
 514             return is_open