UTI: Add Summary to News
[csit.git] / resources / tools / dash / app / pal / news / layout.py
1 # Copyright (c) 2022 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Plotly Dash HTML layout override.
15 """
16
17 import logging
18 import pandas as pd
19 import dash_bootstrap_components as dbc
20
21 from flask import Flask
22 from dash import dcc
23 from dash import html
24 from dash import callback_context
25 from dash import Input, Output, State
26 from yaml import load, FullLoader, YAMLError
27 from copy import deepcopy
28
29 from ..data.data import Data
30 from ..utils.constants import Constants as C
31 from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url, \
32     get_ttypes, get_cadences, get_test_beds, get_job, generate_options, \
33     set_job_params
34 from ..utils.url_processing import url_decode
35 from ..data.data import Data
36 from .tables import table_news, table_summary
37
38
39 class Layout:
40     """The layout of the dash app and the callbacks.
41     """
42
43     def __init__(self, app: Flask, html_layout_file: str, data_spec_file: str,
44         tooltip_file: str) -> None:
45         """Initialization:
46         - save the input parameters,
47         - read and pre-process the data,
48         - prepare data for the control panel,
49         - read HTML layout file,
50         - read tooltips from the tooltip file.
51
52         :param app: Flask application running the dash application.
53         :param html_layout_file: Path and name of the file specifying the HTML
54             layout of the dash application.
55         :param data_spec_file: Path and name of the file specifying the data to
56             be read from parquets for this application.
57         :param tooltip_file: Path and name of the yaml file specifying the
58             tooltips.
59         :type app: Flask
60         :type html_layout_file: str
61         :type data_spec_file: str
62         :type tooltip_file: str
63         """
64
65         # Inputs
66         self._app = app
67         self._html_layout_file = html_layout_file
68         self._data_spec_file = data_spec_file
69         self._tooltip_file = tooltip_file
70
71         # Read the data:
72         data_stats, data_mrr, data_ndrpdr = Data(
73             data_spec_file=self._data_spec_file,
74             debug=True
75         ).read_stats(days=C.NEWS_TIME_PERIOD)
76
77         df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
78
79         # Prepare information for the control panel:
80         self._jobs = sorted(list(df_tst_info["job"].unique()))
81         d_job_info = {
82             "job": list(),
83             "dut": list(),
84             "ttype": list(),
85             "cadence": list(),
86             "tbed": list()
87         }
88         for job in self._jobs:
89             lst_job = job.split("-")
90             d_job_info["job"].append(job)
91             d_job_info["dut"].append(lst_job[1])
92             d_job_info["ttype"].append(lst_job[3])
93             d_job_info["cadence"].append(lst_job[4])
94             d_job_info["tbed"].append("-".join(lst_job[-2:]))
95         self.job_info = pd.DataFrame.from_dict(d_job_info)
96
97         self._default = set_job_params(self.job_info, C.NEWS_DEFAULT_JOB)
98
99         # Pre-process the data:
100
101         def _create_test_name(test: str) -> str:
102             lst_tst = test.split(".")
103             suite = lst_tst[-2].replace("2n1l-", "").replace("1n1l-", "").\
104                 replace("2n-", "")
105             return f"{suite.split('-')[0]}-{lst_tst[-1]}"
106
107         def _get_rindex(array: list, itm: any) -> int:
108             return len(array) - 1 - array[::-1].index(itm)
109
110         tst_info = {
111             "job": list(),
112             "build": list(),
113             "start": list(),
114             "dut_type": list(),
115             "dut_version": list(),
116             "hosts": list(),
117             "failed": list(),
118             "regressions": list(),
119             "progressions": list()
120         }
121         logging.debug("Processing jobs ...")
122         for job in self._jobs:
123             logging.debug(f"+ {job}")
124             # Create lists of failed tests:
125             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
126             last_build = max(df_job["build"].unique())
127             df_build = df_job.loc[(df_job["build"] == last_build)]
128             tst_info["job"].append(job)
129             tst_info["build"].append(last_build)
130             tst_info["start"].append(data_stats.loc[
131                 (data_stats["job"] == job) &
132                 (data_stats["build"] == last_build)
133             ]["start_time"].iloc[-1].strftime('%Y-%m-%d %H:%M'))
134             tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
135             tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
136             tst_info["hosts"].append(df_build["hosts"].iloc[-1])
137             failed_tests = df_build.loc[(df_build["passed"] == False)]\
138                 ["test_id"].to_list()
139             l_failed = list()
140             try:
141                 for tst in failed_tests:
142                     l_failed.append(_create_test_name(tst))
143             except KeyError:
144                 l_failed = list()
145             tst_info["failed"].append(sorted(l_failed))
146
147             # Create lists of regressions and progressions:
148             l_reg = list()
149             l_prog = list()
150
151             tests = df_job["test_id"].unique()
152             for test in tests:
153                 tst_data = df_job.loc[df_job["test_id"] == test].sort_values(
154                     by="start_time", ignore_index=True)
155                 x_axis = tst_data["start_time"].tolist()
156                 if "-ndrpdr" in test:
157                     tst_data = tst_data.dropna(
158                         subset=["result_pdr_lower_rate_value", ]
159                     )
160                     if tst_data.empty:
161                         continue
162                     try:
163                         anomalies, _, _ = classify_anomalies({
164                             k: v for k, v in zip(
165                                 x_axis,
166                                 tst_data["result_ndr_lower_rate_value"].tolist()
167                             )
168                         })
169                     except ValueError:
170                         continue
171                     if "progression" in anomalies:
172                         l_prog.append((
173                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
174                             x_axis[_get_rindex(anomalies, "progression")]
175                         ))
176                     if "regression" in anomalies:
177                         l_reg.append((
178                             _create_test_name(test).replace("-ndrpdr", "-ndr"),
179                             x_axis[_get_rindex(anomalies, "regression")]
180                         ))
181                     try:
182                         anomalies, _, _ = classify_anomalies({
183                             k: v for k, v in zip(
184                                 x_axis,
185                                 tst_data["result_pdr_lower_rate_value"].tolist()
186                             )
187                         })
188                     except ValueError:
189                         continue
190                     if "progression" in anomalies:
191                         l_prog.append((
192                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
193                             x_axis[_get_rindex(anomalies, "progression")]
194                         ))
195                     if "regression" in anomalies:
196                         l_reg.append((
197                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
198                             x_axis[_get_rindex(anomalies, "regression")]
199                         ))
200                 else:  # mrr
201                     tst_data = tst_data.dropna(
202                         subset=["result_receive_rate_rate_avg", ]
203                     )
204                     if tst_data.empty:
205                         continue
206                     try:
207                         anomalies, _, _ = classify_anomalies({
208                             k: v for k, v in zip(
209                                 x_axis,
210                                 tst_data["result_receive_rate_rate_avg"].\
211                                     tolist()
212                             )
213                         })
214                     except ValueError:
215                         continue
216                     if "progression" in anomalies:
217                         l_prog.append((
218                             _create_test_name(test),
219                             x_axis[_get_rindex(anomalies, "progression")]
220                         ))
221                     if "regression" in anomalies:
222                         l_reg.append((
223                             _create_test_name(test),
224                             x_axis[_get_rindex(anomalies, "regression")]
225                         ))
226
227             tst_info["regressions"].append(
228                 sorted(l_reg, key=lambda k: k[1], reverse=True))
229             tst_info["progressions"].append(
230                 sorted(l_prog, key=lambda k: k[1], reverse=True))
231
232         self._data = pd.DataFrame.from_dict(tst_info)
233
234         # Read from files:
235         self._html_layout = str()
236         self._tooltips = dict()
237
238         try:
239             with open(self._html_layout_file, "r") as file_read:
240                 self._html_layout = file_read.read()
241         except IOError as err:
242             raise RuntimeError(
243                 f"Not possible to open the file {self._html_layout_file}\n{err}"
244             )
245
246         try:
247             with open(self._tooltip_file, "r") as file_read:
248                 self._tooltips = load(file_read, Loader=FullLoader)
249         except IOError as err:
250             logging.warning(
251                 f"Not possible to open the file {self._tooltip_file}\n{err}"
252             )
253         except YAMLError as err:
254             logging.warning(
255                 f"An error occurred while parsing the specification file "
256                 f"{self._tooltip_file}\n{err}"
257             )
258
259         self._default_tab_failed = \
260             table_news(self.data, self._default["job"], C.NEWS_TIME_PERIOD)
261
262         # Callbacks:
263         if self._app is not None and hasattr(self, 'callbacks'):
264             self.callbacks(self._app)
265
266     @property
267     def html_layout(self) -> dict:
268         return self._html_layout
269
270     @property
271     def data(self) -> pd.DataFrame:
272         return self._data
273
274     @property
275     def default(self) -> dict:
276         return self._default
277
278     def add_content(self):
279         """Top level method which generated the web page.
280
281         It generates:
282         - Store for user input data,
283         - Navigation bar,
284         - Main area with control panel and ploting area.
285
286         If no HTML layout is provided, an error message is displayed instead.
287
288         :returns: The HTML div with the whole page.
289         :rtype: html.Div
290         """
291
292         if self.html_layout:
293             return html.Div(
294                 id="div-main",
295                 children=[
296                     dcc.Store(id="control-panel"),
297                     dcc.Location(id="url", refresh=False),
298                     dbc.Row(
299                         id="row-navbar",
300                         class_name="g-0",
301                         children=[
302                             self._add_navbar(),
303                         ]
304                     ),
305                     dbc.Row(
306                         id="row-main",
307                         class_name="g-0",
308                         children=[
309                             self._add_ctrl_col(),
310                             self._add_plotting_col(),
311                         ]
312                     )
313                 ]
314             )
315         else:
316             return html.Div(
317                 id="div-main-error",
318                 children=[
319                     dbc.Alert(
320                         [
321                             "An Error Occured",
322                         ],
323                         color="danger",
324                     ),
325                 ]
326             )
327
328     def _add_navbar(self):
329         """Add nav element with navigation panel. It is placed on the top.
330
331         :returns: Navigation bar.
332         :rtype: dbc.NavbarSimple
333         """
334
335         return dbc.NavbarSimple(
336             id="navbarsimple-main",
337             children=[
338                 dbc.NavItem(
339                     dbc.NavLink(
340                         "Continuous Performance News",
341                         disabled=True,
342                         external_link=True,
343                         href="#"
344                     )
345                 )
346             ],
347             brand="Dashboard",
348             brand_href="/",
349             brand_external_link=True,
350             class_name="p-2",
351             fluid=True,
352         )
353
354     def _add_ctrl_col(self) -> dbc.Col:
355         """Add column with control panel. It is placed on the left side.
356
357         :returns: Column with the control panel.
358         :rtype: dbc.Col
359         """
360
361         return dbc.Col(
362             id="col-controls",
363             children=[
364                 self._add_ctrl_panel(),
365             ],
366         )
367
368     def _add_plotting_col(self) -> dbc.Col:
369         """Add column with tables. It is placed on the right side.
370
371         :returns: Column with tables.
372         :rtype: dbc.Col
373         """
374
375         return dbc.Col(
376             id="col-plotting-area",
377             children=[
378                 dbc.Row(  # Failed tests
379                     id="row-table-failed",
380                     class_name="g-0 p-2",
381                     children=self._default_tab_failed
382                 ),
383                 dbc.Row(
384                     class_name="g-0 p-2",
385                     align="center",
386                     justify="start",
387                     children=[
388                         dbc.InputGroup(
389                             class_name="me-1",
390                             children=[
391                                 dbc.InputGroupText(
392                                     style=C.URL_STYLE,
393                                     children=show_tooltip(
394                                         self._tooltips,
395                                         "help-url", "URL",
396                                         "input-url"
397                                     )
398                                 ),
399                                 dbc.Input(
400                                     id="input-url",
401                                     readonly=True,
402                                     type="url",
403                                     style=C.URL_STYLE,
404                                     value=""
405                                 )
406                             ]
407                         )
408                     ]
409                 )
410             ],
411             width=9,
412         )
413
414     def _add_ctrl_panel(self) -> dbc.Row:
415         """Add control panel.
416
417         :returns: Control panel.
418         :rtype: dbc.Row
419         """
420         return dbc.Row(
421             id="row-ctrl-panel",
422             class_name="g-0",
423             children=[
424                 dbc.Row(
425                     class_name="g-0 p-2",
426                     children=[
427                         dbc.Row(
428                             class_name="gy-1",
429                             children=[
430                                 dbc.Label(
431                                     class_name="p-0",
432                                     children=show_tooltip(self._tooltips,
433                                         "help-dut", "Device under Test")
434                                 ),
435                                 dbc.Row(
436                                     dbc.RadioItems(
437                                         id="ri-duts",
438                                         inline=True,
439                                         value=self.default["dut"],
440                                         options=self.default["duts"]
441                                     )
442                                 )
443                             ]
444                         ),
445                         dbc.Row(
446                             class_name="gy-1",
447                             children=[
448                                 dbc.Label(
449                                     class_name="p-0",
450                                     children=show_tooltip(self._tooltips,
451                                         "help-ttype", "Test Type"),
452                                 ),
453                                 dbc.RadioItems(
454                                     id="ri-ttypes",
455                                     inline=True,
456                                     value=self.default["ttype"],
457                                     options=self.default["ttypes"]
458                                 )
459                             ]
460                         ),
461                         dbc.Row(
462                             class_name="gy-1",
463                             children=[
464                                 dbc.Label(
465                                     class_name="p-0",
466                                     children=show_tooltip(self._tooltips,
467                                         "help-cadence", "Cadence"),
468                                 ),
469                                 dbc.RadioItems(
470                                     id="ri-cadences",
471                                     inline=True,
472                                     value=self.default["cadence"],
473                                     options=self.default["cadences"]
474                                 )
475                             ]
476                         ),
477                         dbc.Row(
478                             class_name="gy-1",
479                             children=[
480                                 dbc.Label(
481                                     class_name="p-0",
482                                     children=show_tooltip(self._tooltips,
483                                         "help-tbed", "Test Bed"),
484                                 ),
485                                 dbc.Select(
486                                     id="dd-tbeds",
487                                     placeholder="Select a test bed...",
488                                     value=self.default["tbed"],
489                                     options=self.default["tbeds"]
490                                 )
491                             ]
492                         ),
493                         dbc.Row(
494                             class_name="gy-1 p-0",
495                             children=[
496                                 dbc.ButtonGroup(
497                                     [
498                                         dbc.Button(
499                                             id="btn-summary",
500                                             children=(
501                                                 f"Show Summary from the last "
502                                                 f"{C.NEWS_SUMMARY_PERIOD} Days"
503                                             ),
504                                             class_name="me-1",
505                                             color="info"
506                                         )
507                                     ],
508                                     size="md",
509                                 )
510                             ]
511                         ),
512                         dbc.Row(
513                             class_name="gy-1",
514                             children=[
515                                 dbc.Alert(
516                                     id="al-job",
517                                     color="info",
518                                     children=self.default["job"]
519                                 )
520                             ]
521                         )
522                     ]
523                 )
524             ]
525         )
526
527     class ControlPanel:
528         """A class representing the control panel.
529         """
530
531         def __init__(self, panel: dict, default: dict) -> None:
532             """Initialisation of the control pannel by default values. If
533             particular values are provided (parameter "panel") they are set
534             afterwards.
535
536             :param panel: Custom values to be set to the control panel.
537             :param default: Default values to be set to the control panel.
538             :type panel: dict
539             :type defaults: dict
540             """
541
542             self._defaults = {
543                 "ri-ttypes-options": default["ttypes"],
544                 "ri-cadences-options": default["cadences"],
545                 "dd-tbeds-options": default["tbeds"],
546                 "ri-duts-value": default["dut"],
547                 "ri-ttypes-value": default["ttype"],
548                 "ri-cadences-value": default["cadence"],
549                 "dd-tbeds-value": default["tbed"],
550                 "al-job-children": default["job"]
551             }
552             self._panel = deepcopy(self._defaults)
553             if panel:
554                 for key in self._defaults:
555                     self._panel[key] = panel[key]
556
557         def set(self, kwargs: dict) -> None:
558             """Set the values of the Control panel.
559
560             :param kwargs: key - value pairs to be set.
561             :type kwargs: dict
562             :raises KeyError: If the key in kwargs is not present in the Control
563                 panel.
564             """
565             for key, val in kwargs.items():
566                 if key in self._panel:
567                     self._panel[key] = val
568                 else:
569                     raise KeyError(f"The key {key} is not defined.")
570
571         @property
572         def defaults(self) -> dict:
573             return self._defaults
574
575         @property
576         def panel(self) -> dict:
577             return self._panel
578
579         def get(self, key: str) -> any:
580             """Returns the value of a key from the Control panel.
581
582             :param key: The key which value should be returned.
583             :type key: str
584             :returns: The value of the key.
585             :rtype: any
586             :raises KeyError: If the key in kwargs is not present in the Control
587                 panel.
588             """
589             return self._panel[key]
590
591         def values(self) -> list:
592             """Returns the values from the Control panel as a list.
593
594             :returns: The values from the Control panel.
595             :rtype: list
596             """
597             return list(self._panel.values())
598
599     def callbacks(self, app):
600         """Callbacks for the whole application.
601
602         :param app: The application.
603         :type app: Flask
604         """
605
606         @app.callback(
607             Output("control-panel", "data"),  # Store
608             Output("row-table-failed", "children"),
609             Output("input-url", "value"),
610             Output("ri-ttypes", "options"),
611             Output("ri-cadences", "options"),
612             Output("dd-tbeds", "options"),
613             Output("ri-duts", "value"),
614             Output("ri-ttypes", "value"),
615             Output("ri-cadences", "value"),
616             Output("dd-tbeds", "value"),
617             Output("al-job", "children"),
618             State("control-panel", "data"),  # Store
619             Input("ri-duts", "value"),
620             Input("ri-ttypes", "value"),
621             Input("ri-cadences", "value"),
622             Input("dd-tbeds", "value"),
623             Input("url", "href"),
624             Input("btn-summary", "n_clicks")
625         )
626         def _update_application(cp_data: dict, dut: str, ttype: str,
627                 cadence:str, tbed: str, href: str, btn_all: int) -> tuple:
628             """Update the application when the event is detected.
629
630             :param cp_data: Current status of the control panel stored in
631                 browser.
632             :param dut: Input - DUT name.
633             :param ttype: Input - Test type.
634             :param cadence: Input - The cadence of the job.
635             :param tbed: Input - The test bed.
636             :param href: Input - The URL provided by the browser.
637             :type cp_data: dict
638             :type dut: str
639             :type ttype: str
640             :type cadence: str
641             :type tbed: str
642             :type href: str
643             :returns: New values for web page elements.
644             :rtype: tuple
645             """
646
647             ctrl_panel = self.ControlPanel(cp_data, self.default)
648
649             # Parse the url:
650             parsed_url = url_decode(href)
651             if parsed_url:
652                 url_params = parsed_url["params"]
653             else:
654                 url_params = None
655
656             show_summary = False
657
658             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
659             if trigger_id == "ri-duts":
660                 ttype_opts = generate_options(get_ttypes(self.job_info, dut))
661                 ttype_val = ttype_opts[0]["value"]
662                 cad_opts = generate_options(
663                     get_cadences(self.job_info, dut, ttype_val))
664                 cad_val = cad_opts[0]["value"]
665                 tbed_opts = generate_options(get_test_beds(
666                     self.job_info, dut, ttype_val, cad_val))
667                 tbed_val = tbed_opts[0]["value"]
668                 ctrl_panel.set({
669                     "ri-duts-value": dut,
670                     "ri-ttypes-options": ttype_opts,
671                     "ri-ttypes-value": ttype_val,
672                     "ri-cadences-options": cad_opts,
673                     "ri-cadences-value": cad_val,
674                     "dd-tbeds-options": tbed_opts,
675                     "dd-tbeds-value": tbed_val
676                 })
677             elif trigger_id == "ri-ttypes":
678                 cad_opts = generate_options(get_cadences(
679                     self.job_info, ctrl_panel.get("ri-duts-value"), ttype))
680                 cad_val = cad_opts[0]["value"]
681                 tbed_opts = generate_options(get_test_beds(
682                     self.job_info, ctrl_panel.get("ri-duts-value"),
683                     ttype, cad_val))
684                 tbed_val = tbed_opts[0]["value"]
685                 ctrl_panel.set({
686                     "ri-ttypes-value": ttype,
687                     "ri-cadences-options": cad_opts,
688                     "ri-cadences-value": cad_val,
689                     "dd-tbeds-options": tbed_opts,
690                     "dd-tbeds-value": tbed_val
691                 })
692             elif trigger_id == "ri-cadences":
693                 tbed_opts = generate_options(get_test_beds(
694                     self.job_info, ctrl_panel.get("ri-duts-value"),
695                     ctrl_panel.get("ri-ttypes-value"), cadence))
696                 tbed_val = tbed_opts[0]["value"]
697                 ctrl_panel.set({
698                     "ri-cadences-value": cadence,
699                     "dd-tbeds-options": tbed_opts,
700                     "dd-tbeds-value": tbed_val
701                 })
702             elif trigger_id == "dd-tbeds":
703                 ctrl_panel.set({
704                     "dd-tbeds-value": tbed
705                 })
706             elif trigger_id == "url":
707                 # TODO: Add verification
708                 if url_params:
709                     new_job = url_params.get("job", list())[0]
710                     if new_job and new_job != "all":
711                         job_params = set_job_params(self.job_info, new_job)
712                         ctrl_panel = self.ControlPanel(None, job_params)
713                     if new_job and new_job == "all":
714                         show_summary = True
715                 else:
716                     ctrl_panel = self.ControlPanel(cp_data, self.default)
717             elif trigger_id == "btn-summary":
718                 show_summary = True
719
720             if show_summary:
721                 ctrl_panel.set({
722                     "al-job-children": \
723                         f"Summary from the last {C.NEWS_SUMMARY_PERIOD} days"
724                 })
725                 job = "all"
726                 tables = table_summary(self.data, self._jobs)
727             else:
728                 job = get_job(
729                     self.job_info,
730                     ctrl_panel.get("ri-duts-value"),
731                     ctrl_panel.get("ri-ttypes-value"),
732                     ctrl_panel.get("ri-cadences-value"),
733                     ctrl_panel.get("dd-tbeds-value")
734                 )
735                 ctrl_panel.set({"al-job-children": job})
736                 tables = table_news(self.data, job, C.NEWS_TIME_PERIOD)
737
738             ret_val = [
739                 ctrl_panel.panel,
740                 tables,
741                 gen_new_url(parsed_url, {"job": job})
742             ]
743             ret_val.extend(ctrl_panel.values())
744             return ret_val