csit.infra.dash/app/cdash/news/layout.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Plotly Dash HTML layout override.
  15 """
  16
  17 import logging
  18 import pandas as pd
  19 import dash_bootstrap_components as dbc
  20
  21 from flask import Flask
  22 from dash import dcc
  23 from dash import html
  24 from dash import callback_context
  25 from dash import Input, Output, State
  26 from yaml import load, FullLoader, YAMLError
  27
  28 from ..data.data import Data
  29 from ..utils.constants import Constants as C
  30 from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url
  31 from ..utils.url_processing import url_decode
  32 from ..data.data import Data
  33 from .tables import table_summary
  34
  35
  36 class Layout:
  37     """The layout of the dash app and the callbacks.
  38     """
  39
  40     def __init__(self, app: Flask, html_layout_file: str, data_spec_file: str,
  41         tooltip_file: str) -> None:
  42         """Initialization:
  43         - save the input parameters,
  44         - read and pre-process the data,
  45         - prepare data for the control panel,
  46         - read HTML layout file,
  47         - read tooltips from the tooltip file.
  48
  49         :param app: Flask application running the dash application.
  50         :param html_layout_file: Path and name of the file specifying the HTML
  51             layout of the dash application.
  52         :param data_spec_file: Path and name of the file specifying the data to
  53             be read from parquets for this application.
  54         :param tooltip_file: Path and name of the yaml file specifying the
  55             tooltips.
  56         :type app: Flask
  57         :type html_layout_file: str
  58         :type data_spec_file: str
  59         :type tooltip_file: str
  60         """
  61
  62         # Inputs
  63         self._app = app
  64         self._html_layout_file = html_layout_file
  65         self._data_spec_file = data_spec_file
  66         self._tooltip_file = tooltip_file
  67
  68         # Read the data:
  69         data_stats, data_mrr, data_ndrpdr = Data(
  70             data_spec_file=self._data_spec_file,
  71             debug=True
  72         ).read_stats(days=C.NEWS_TIME_PERIOD)
  73
  74         df_tst_info = pd.concat(
  75             [data_mrr, data_ndrpdr],
  76             ignore_index=True,
  77             copy=False
  78         )
  79
  80         # Prepare information for the control panel:
  81         self._jobs = sorted(list(df_tst_info["job"].unique()))
  82         d_job_info = {
  83             "job": list(),
  84             "dut": list(),
  85             "ttype": list(),
  86             "cadence": list(),
  87             "tbed": list()
  88         }
  89         for job in self._jobs:
  90             lst_job = job.split("-")
  91             d_job_info["job"].append(job)
  92             d_job_info["dut"].append(lst_job[1])
  93             d_job_info["ttype"].append(lst_job[3])
  94             d_job_info["cadence"].append(lst_job[4])
  95             d_job_info["tbed"].append("-".join(lst_job[-2:]))
  96         self.job_info = pd.DataFrame.from_dict(d_job_info)
  97
  98         # Pre-process the data:
  99
 100         def _create_test_name(test: str) -> str:
 101             lst_tst = test.split(".")
 102             suite = lst_tst[-2].replace("2n1l-", "").replace("1n1l-", "").\
 103                 replace("2n-", "")
 104             return f"{suite.split('-')[0]}-{lst_tst[-1]}"
 105
 106         def _get_rindex(array: list, itm: any) -> int:
 107             return len(array) - 1 - array[::-1].index(itm)
 108
 109         tst_info = {
 110             "job": list(),
 111             "build": list(),
 112             "start": list(),
 113             "dut_type": list(),
 114             "dut_version": list(),
 115             "hosts": list(),
 116             "failed": list(),
 117             "regressions": list(),
 118             "progressions": list()
 119         }
 120         for job in self._jobs:
 121             # Create lists of failed tests:
 122             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
 123             last_build = str(max(pd.to_numeric(df_job["build"].unique())))
 124             df_build = df_job.loc[(df_job["build"] == last_build)]
 125             tst_info["job"].append(job)
 126             tst_info["build"].append(last_build)
 127             tst_info["start"].append(data_stats.loc[
 128                 (data_stats["job"] == job) &
 129                 (data_stats["build"] == last_build)
 130             ]["start_time"].iloc[-1].strftime('%Y-%m-%d %H:%M'))
 131             tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
 132             tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
 133             tst_info["hosts"].append(df_build["hosts"].iloc[-1])
 134             failed_tests = df_build.loc[(df_build["passed"] == False)]\
 135                 ["test_id"].to_list()
 136             l_failed = list()
 137             try:
 138                 for tst in failed_tests:
 139                     l_failed.append(_create_test_name(tst))
 140             except KeyError:
 141                 l_failed = list()
 142             tst_info["failed"].append(sorted(l_failed))
 143
 144             # Create lists of regressions and progressions:
 145             l_reg = list()
 146             l_prog = list()
 147
 148             tests = df_job["test_id"].unique()
 149             for test in tests:
 150                 tst_data = df_job.loc[df_job["test_id"] == test].sort_values(
 151                     by="start_time", ignore_index=True)
 152                 x_axis = tst_data["start_time"].tolist()
 153                 if "-ndrpdr" in test:
 154                     tst_data = tst_data.dropna(
 155                         subset=["result_pdr_lower_rate_value", ]
 156                     )
 157                     if tst_data.empty:
 158                         continue
 159                     try:
 160                         anomalies, _, _ = classify_anomalies({
 161                             k: v for k, v in zip(
 162                                 x_axis,
 163                                 tst_data["result_ndr_lower_rate_value"].tolist()
 164                             )
 165                         })
 166                     except ValueError:
 167                         continue
 168                     if "progression" in anomalies:
 169                         l_prog.append((
 170                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 171                             x_axis[_get_rindex(anomalies, "progression")]
 172                         ))
 173                     if "regression" in anomalies:
 174                         l_reg.append((
 175                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
 176                             x_axis[_get_rindex(anomalies, "regression")]
 177                         ))
 178                     try:
 179                         anomalies, _, _ = classify_anomalies({
 180                             k: v for k, v in zip(
 181                                 x_axis,
 182                                 tst_data["result_pdr_lower_rate_value"].tolist()
 183                             )
 184                         })
 185                     except ValueError:
 186                         continue
 187                     if "progression" in anomalies:
 188                         l_prog.append((
 189                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 190                             x_axis[_get_rindex(anomalies, "progression")]
 191                         ))
 192                     if "regression" in anomalies:
 193                         l_reg.append((
 194                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
 195                             x_axis[_get_rindex(anomalies, "regression")]
 196                         ))
 197                 else:  # mrr
 198                     tst_data = tst_data.dropna(
 199                         subset=["result_receive_rate_rate_avg", ]
 200                     )
 201                     if tst_data.empty:
 202                         continue
 203                     try:
 204                         anomalies, _, _ = classify_anomalies({
 205                             k: v for k, v in zip(
 206                                 x_axis,
 207                                 tst_data["result_receive_rate_rate_avg"].\
 208                                     tolist()
 209                             )
 210                         })
 211                     except ValueError:
 212                         continue
 213                     if "progression" in anomalies:
 214                         l_prog.append((
 215                             _create_test_name(test),
 216                             x_axis[_get_rindex(anomalies, "progression")]
 217                         ))
 218                     if "regression" in anomalies:
 219                         l_reg.append((
 220                             _create_test_name(test),
 221                             x_axis[_get_rindex(anomalies, "regression")]
 222                         ))
 223
 224             tst_info["regressions"].append(
 225                 sorted(l_reg, key=lambda k: k[1], reverse=True))
 226             tst_info["progressions"].append(
 227                 sorted(l_prog, key=lambda k: k[1], reverse=True))
 228
 229         self._data = pd.DataFrame.from_dict(tst_info)
 230
 231         # Read from files:
 232         self._html_layout = str()
 233         self._tooltips = dict()
 234
 235         try:
 236             with open(self._html_layout_file, "r") as file_read:
 237                 self._html_layout = file_read.read()
 238         except IOError as err:
 239             raise RuntimeError(
 240                 f"Not possible to open the file {self._html_layout_file}\n{err}"
 241             )
 242
 243         try:
 244             with open(self._tooltip_file, "r") as file_read:
 245                 self._tooltips = load(file_read, Loader=FullLoader)
 246         except IOError as err:
 247             logging.warning(
 248                 f"Not possible to open the file {self._tooltip_file}\n{err}"
 249             )
 250         except YAMLError as err:
 251             logging.warning(
 252                 f"An error occurred while parsing the specification file "
 253                 f"{self._tooltip_file}\n{err}"
 254             )
 255
 256         self._default_period = C.NEWS_SHORT
 257         self._default_active = (False, True, False)
 258
 259         # Callbacks:
 260         if self._app is not None and hasattr(self, 'callbacks'):
 261             self.callbacks(self._app)
 262
 263     @property
 264     def html_layout(self) -> dict:
 265         return self._html_layout
 266
 267     def add_content(self):
 268         """Top level method which generated the web page.
 269
 270         It generates:
 271         - Store for user input data,
 272         - Navigation bar,
 273         - Main area with control panel and ploting area.
 274
 275         If no HTML layout is provided, an error message is displayed instead.
 276
 277         :returns: The HTML div with the whole page.
 278         :rtype: html.Div
 279         """
 280
 281         if self.html_layout:
 282             return html.Div(
 283                 id="div-main",
 284                 className="small",
 285                 children=[
 286                     dcc.Location(id="url", refresh=False),
 287                     dbc.Row(
 288                         id="row-navbar",
 289                         class_name="g-0",
 290                         children=[
 291                             self._add_navbar()
 292                         ]
 293                     ),
 294                     dbc.Row(
 295                         id="row-main",
 296                         class_name="g-0",
 297                         children=[
 298                             self._add_ctrl_col(),
 299                             self._add_plotting_col()
 300                         ]
 301                     )
 302                 ]
 303             )
 304         else:
 305             return html.Div(
 306                 id="div-main-error",
 307                 children=[
 308                     dbc.Alert(
 309                         [
 310                             "An Error Occured"
 311                         ],
 312                         color="danger"
 313                     )
 314                 ]
 315             )
 316
 317     def _add_navbar(self):
 318         """Add nav element with navigation panel. It is placed on the top.
 319
 320         :returns: Navigation bar.
 321         :rtype: dbc.NavbarSimple
 322         """
 323
 324         return dbc.NavbarSimple(
 325             id="navbarsimple-main",
 326             children=[
 327                 dbc.NavItem(
 328                     dbc.NavLink(
 329                         C.NEWS_TITLE,
 330                         disabled=True,
 331                         external_link=True,
 332                         href="#"
 333                     )
 334                 )
 335             ],
 336             brand=C.BRAND,
 337             brand_href="/",
 338             brand_external_link=True,
 339             class_name="p-2",
 340             fluid=True
 341         )
 342
 343     def _add_ctrl_col(self) -> dbc.Col:
 344         """Add column with control panel. It is placed on the left side.
 345
 346         :returns: Column with the control panel.
 347         :rtype: dbc.Col
 348         """
 349         return dbc.Col([
 350             html.Div(
 351                 children=self._add_ctrl_panel(),
 352                 className="sticky-top"
 353             )
 354         ])
 355
 356     def _add_plotting_col(self) -> dbc.Col:
 357         """Add column with tables. It is placed on the right side.
 358
 359         :returns: Column with tables.
 360         :rtype: dbc.Col
 361         """
 362         return dbc.Col(
 363             id="col-plotting-area",
 364             children=[
 365                 dbc.Spinner(
 366                     children=[
 367                         dbc.Row(
 368                             id="plotting-area",
 369                             class_name="g-0 p-0",
 370                             children=[
 371                                 C.PLACEHOLDER
 372                             ]
 373                         )
 374                     ]
 375                 )
 376             ],
 377             width=9
 378         )
 379
 380     def _add_ctrl_panel(self) -> list:
 381         """Add control panel.
 382
 383         :returns: Control panel.
 384         :rtype: list
 385         """
 386         return [
 387             dbc.Row(
 388                 class_name="g-0 p-1",
 389                 children=[
 390                     dbc.ButtonGroup(
 391                         id="bg-time-period",
 392                         children=[
 393                             dbc.Button(
 394                                 id="period-last",
 395                                 children="Last Run",
 396                                 className="me-1",
 397                                 outline=True,
 398                                 color="info"
 399                             ),
 400                             dbc.Button(
 401                                 id="period-short",
 402                                 children=f"Last {C.NEWS_SHORT} Runs",
 403                                 className="me-1",
 404                                 outline=True,
 405                                 active=True,
 406                                 color="info"
 407                             ),
 408                             dbc.Button(
 409                                 id="period-long",
 410                                 children="All Runs",
 411                                 className="me-1",
 412                                 outline=True,
 413                                 color="info"
 414                             )
 415                         ]
 416                     )
 417                 ]
 418             )
 419         ]
 420
 421     def _get_plotting_area(
 422             self,
 423             period: int,
 424             url: str
 425         ) -> list:
 426         """Generate the plotting area with all its content.
 427
 428         :param period: The time period for summary tables.
 429         :param url: URL to be displayed in the modal window.
 430         :type period: int
 431         :type url: str
 432         :returns: The content of the plotting area.
 433         :rtype: list
 434         """
 435         return [
 436             dbc.Row(
 437                 id="row-table",
 438                 class_name="g-0 p-1",
 439                 children=table_summary(self._data, self._jobs, period)
 440             ),
 441             dbc.Row(
 442                 [
 443                     dbc.Col([html.Div(
 444                         [
 445                             dbc.Button(
 446                                 id="plot-btn-url",
 447                                 children="Show URL",
 448                                 class_name="me-1",
 449                                 color="info",
 450                                 style={
 451                                     "text-transform": "none",
 452                                     "padding": "0rem 1rem"
 453                                 }
 454                             ),
 455                             dbc.Modal(
 456                                 [
 457                                     dbc.ModalHeader(dbc.ModalTitle("URL")),
 458                                     dbc.ModalBody(url)
 459                                 ],
 460                                 id="plot-mod-url",
 461                                 size="xl",
 462                                 is_open=False,
 463                                 scrollable=True
 464                             )
 465                         ],
 466                         className=\
 467                             "d-grid gap-0 d-md-flex justify-content-md-end"
 468                     )])
 469                 ],
 470                 class_name="g-0 p-0"
 471             )
 472         ]
 473
 474     def callbacks(self, app):
 475         """Callbacks for the whole application.
 476
 477         :param app: The application.
 478         :type app: Flask
 479         """
 480
 481         @app.callback(
 482             Output("plotting-area", "children"),
 483             Output("period-last", "active"),
 484             Output("period-short", "active"),
 485             Output("period-long", "active"),
 486             Input("url", "href"),
 487             Input("period-last", "n_clicks"),
 488             Input("period-short", "n_clicks"),
 489             Input("period-long", "n_clicks")
 490         )
 491         def _update_application(href: str, *_) -> tuple:
 492             """Update the application when the event is detected.
 493
 494             :returns: New values for web page elements.
 495             :rtype: tuple
 496             """
 497
 498             periods = {
 499                 "period-last": C.NEWS_LAST,
 500                 "period-short": C.NEWS_SHORT,
 501                 "period-long": C.NEWS_LONG
 502             }
 503             actives = {
 504                 "period-last": (True, False, False),
 505                 "period-short": (False, True, False),
 506                 "period-long": (False, False, True)
 507             }
 508
 509             # Parse the url:
 510             parsed_url = url_decode(href)
 511             if parsed_url:
 512                 url_params = parsed_url["params"]
 513             else:
 514                 url_params = None
 515
 516             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
 517             if trigger_id == "url" and url_params:
 518                 trigger_id = url_params.get("period", list())[0]
 519
 520             ret_val = [
 521                 self._get_plotting_area(
 522                     periods.get(trigger_id, self._default_period),
 523                     gen_new_url(parsed_url, {"period": trigger_id})
 524                 )
 525             ]
 526             ret_val.extend(actives.get(trigger_id, self._default_active))
 527             return ret_val
 528
 529         @app.callback(
 530             Output("plot-mod-url", "is_open"),
 531             [Input("plot-btn-url", "n_clicks")],
 532             [State("plot-mod-url", "is_open")],
 533         )
 534         def toggle_plot_mod_url(n, is_open):
 535             """Toggle the modal window with url.
 536             """
 537             if n:
 538                 return not is_open
 539             return is_open