1b2aebe332c4d4126b517c59837862108ab503eb
[csit.git] / resources / tools / dash / app / pal / stats / layout.py
1 # Copyright (c) 2022 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Plotly Dash HTML layout override.
15 """
16
17 import logging
18 import pandas as pd
19 import dash_bootstrap_components as dbc
20
21 from flask import Flask
22 from dash import dcc
23 from dash import html
24 from dash import callback_context, no_update
25 from dash import Input, Output, State
26 from dash.exceptions import PreventUpdate
27 from yaml import load, FullLoader, YAMLError
28 from datetime import datetime, timedelta
29 from copy import deepcopy
30
31 from ..data.data import Data
32 from .graphs import graph_statistics, select_data
33
34
35 class Layout:
36     """
37     """
38
39     DEFAULT_JOB = "csit-vpp-perf-mrr-daily-master-2n-icx"
40
41     def __init__(self, app: Flask, html_layout_file: str, spec_file: str,
42         graph_layout_file: str, data_spec_file: str, tooltip_file: str,
43         time_period: int=None) -> None:
44         """
45         """
46
47         # Inputs
48         self._app = app
49         self._html_layout_file = html_layout_file
50         self._spec_file = spec_file
51         self._graph_layout_file = graph_layout_file
52         self._data_spec_file = data_spec_file
53         self._tooltip_file = tooltip_file
54         self._time_period = time_period
55
56         # Read the data:
57         data_stats, data_mrr, data_ndrpdr = Data(
58             data_spec_file=self._data_spec_file,
59             debug=True
60         ).read_stats(days=self._time_period)
61
62         df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
63
64         # Pre-process the data:
65         data_stats = data_stats[~data_stats.job.str.contains("-verify-")]
66         data_stats = data_stats[~data_stats.job.str.contains("-coverage-")]
67         data_stats = data_stats[~data_stats.job.str.contains("-iterative-")]
68         data_stats = data_stats[["job", "build", "start_time", "duration"]]
69
70         data_time_period = \
71             (datetime.utcnow() - data_stats["start_time"].min()).days
72         if self._time_period > data_time_period:
73             self._time_period = data_time_period
74
75         jobs = sorted(list(data_stats["job"].unique()))
76         job_info = {
77             "job": list(),
78             "dut": list(),
79             "ttype": list(),
80             "cadence": list(),
81             "tbed": list()
82         }
83         for job in jobs:
84             lst_job = job.split("-")
85             job_info["job"].append(job)
86             job_info["dut"].append(lst_job[1])
87             job_info["ttype"].append(lst_job[3])
88             job_info["cadence"].append(lst_job[4])
89             job_info["tbed"].append("-".join(lst_job[-2:]))
90         self.df_job_info = pd.DataFrame.from_dict(job_info)
91
92         lst_job = self.DEFAULT_JOB.split("-")
93         self._default = {
94             "job": self.DEFAULT_JOB,
95             "dut": lst_job[1],
96             "ttype": lst_job[3],
97             "cadence": lst_job[4],
98             "tbed": "-".join(lst_job[-2:]),
99             "duts": self._generate_options(self._get_duts()),
100             "ttypes": self._generate_options(self._get_ttypes(lst_job[1])),
101             "cadences": self._generate_options(self._get_cadences(
102                 lst_job[1], lst_job[3])),
103             "tbeds": self._generate_options(self._get_test_beds(
104                 lst_job[1], lst_job[3], lst_job[4]))
105         }
106
107         tst_info = {
108             "job": list(),
109             "build": list(),
110             "dut_type": list(),
111             "dut_version": list(),
112             "hosts": list(),
113             "passed": list(),
114             "failed": list()
115         }
116         for job in jobs:
117             # TODO: Add list of failed tests for each build
118             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
119             builds = df_job["build"].unique()
120             for build in builds:
121                 df_build = df_job.loc[(df_job["build"] == build)]
122                 tst_info["job"].append(job)
123                 tst_info["build"].append(build)
124                 tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
125                 tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
126                 tst_info["hosts"].append(df_build["hosts"].iloc[-1])
127                 try:
128                     passed = df_build.value_counts(subset='passed')[True]
129                 except KeyError:
130                     passed = 0
131                 try:
132                     failed = df_build.value_counts(subset='passed')[False]
133                 except KeyError:
134                     failed = 0
135                 tst_info["passed"].append(passed)
136                 tst_info["failed"].append(failed)
137
138         self._data = data_stats.merge(pd.DataFrame.from_dict(tst_info))
139
140         # Read from files:
141         self._html_layout = ""
142         self._graph_layout = None
143         self._tooltips = dict()
144
145         try:
146             with open(self._html_layout_file, "r") as file_read:
147                 self._html_layout = file_read.read()
148         except IOError as err:
149             raise RuntimeError(
150                 f"Not possible to open the file {self._html_layout_file}\n{err}"
151             )
152
153         try:
154             with open(self._graph_layout_file, "r") as file_read:
155                 self._graph_layout = load(file_read, Loader=FullLoader)
156         except IOError as err:
157             raise RuntimeError(
158                 f"Not possible to open the file {self._graph_layout_file}\n"
159                 f"{err}"
160             )
161         except YAMLError as err:
162             raise RuntimeError(
163                 f"An error occurred while parsing the specification file "
164                 f"{self._graph_layout_file}\n{err}"
165             )
166
167         try:
168             with open(self._tooltip_file, "r") as file_read:
169                 self._tooltips = load(file_read, Loader=FullLoader)
170         except IOError as err:
171             logging.warning(
172                 f"Not possible to open the file {self._tooltip_file}\n{err}"
173             )
174         except YAMLError as err:
175             logging.warning(
176                 f"An error occurred while parsing the specification file "
177                 f"{self._tooltip_file}\n{err}"
178             )
179
180
181         self._default_fig_passed, self._default_fig_duration = graph_statistics(
182             self.data, self._default["job"], self.layout
183         )
184
185         # Callbacks:
186         if self._app is not None and hasattr(self, 'callbacks'):
187             self.callbacks(self._app)
188
189     @property
190     def html_layout(self) -> dict:
191         return self._html_layout
192
193     @property
194     def data(self) -> pd.DataFrame:
195         return self._data
196
197     @property
198     def layout(self) -> dict:
199         return self._graph_layout
200
201     @property
202     def time_period(self) -> int:
203         return self._time_period
204
205     @property
206     def default(self) -> any:
207         return self._default
208
209     def _get_duts(self) -> list:
210         """
211         """
212         return sorted(list(self.df_job_info["dut"].unique()))
213
214     def _get_ttypes(self, dut: str) -> list:
215         """
216         """
217         return sorted(list(self.df_job_info.loc[(
218             self.df_job_info["dut"] == dut
219         )]["ttype"].unique()))
220
221     def _get_cadences(self, dut: str, ttype: str) -> list:
222         """
223         """
224         return sorted(list(self.df_job_info.loc[(
225             (self.df_job_info["dut"] == dut) &
226             (self.df_job_info["ttype"] == ttype)
227         )]["cadence"].unique()))
228
229     def _get_test_beds(self, dut: str, ttype: str, cadence: str) -> list:
230         """
231         """
232         return sorted(list(self.df_job_info.loc[(
233             (self.df_job_info["dut"] == dut) &
234             (self.df_job_info["ttype"] == ttype) &
235             (self.df_job_info["cadence"] == cadence)
236         )]["tbed"].unique()))
237
238     def _get_job(self, dut, ttype, cadence, testbed):
239         """Get the name of a job defined by dut, ttype, cadence, testbed.
240
241         Input information comes from control panel.
242         """
243         return self.df_job_info.loc[(
244             (self.df_job_info["dut"] == dut) &
245             (self.df_job_info["ttype"] == ttype) &
246             (self.df_job_info["cadence"] == cadence) &
247             (self.df_job_info["tbed"] == testbed)
248         )]["job"].item()
249
250     def _show_tooltip(self, id: str, title: str) -> list:
251         """
252         """
253         return [
254             f"{title} ",
255             dbc.Badge(
256                 id=id,
257                 children="?",
258                 pill=True,
259                 color="white",
260                 text_color="info",
261                 class_name="border ms-1",
262             ),
263             dbc.Tooltip(
264                 children=self._tooltips.get(id, str()),
265                 target=id,
266                 placement="auto"
267             )
268         ]
269
270     def add_content(self):
271         """
272         """
273         if self.html_layout:
274             return html.Div(
275                 id="div-main",
276                 children=[
277                     dcc.Store(
278                         id="control-panel"
279                     ),
280                     dbc.Row(
281                         id="row-navbar",
282                         class_name="g-0",
283                         children=[
284                             self._add_navbar(),
285                         ]
286                     ),
287                     dcc.Loading(
288                         dbc.Offcanvas(
289                             class_name="w-25",
290                             id="offcanvas-metadata",
291                             title="Detailed Information",
292                             placement="end",
293                             is_open=False,
294                             children=[
295                                 dbc.Row(id="row-metadata")
296                             ]
297                         )
298                     ),
299                     dbc.Row(
300                         id="row-main",
301                         class_name="g-0",
302                         children=[
303                             self._add_ctrl_col(),
304                             self._add_plotting_col(),
305                         ]
306                     )
307                 ]
308             )
309         else:
310             return html.Div(
311                 id="div-main-error",
312                 children=[
313                     dbc.Alert(
314                         [
315                             "An Error Occured",
316                         ],
317                         color="danger",
318                     ),
319                 ]
320             )
321
322     def _add_navbar(self):
323         """Add nav element with navigation panel. It is placed on the top.
324         """
325         return dbc.NavbarSimple(
326             id="navbarsimple-main",
327             children=[
328                 dbc.NavItem(
329                     dbc.NavLink(
330                         "Continuous Performance Statistics",
331                         disabled=True,
332                         external_link=True,
333                         href="#"
334                     )
335                 )
336             ],
337             brand="Dashboard",
338             brand_href="/",
339             brand_external_link=True,
340             class_name="p-2",
341             fluid=True,
342         )
343
344     def _add_ctrl_col(self) -> dbc.Col:
345         """Add column with controls. It is placed on the left side.
346         """
347         return dbc.Col(
348             id="col-controls",
349             children=[
350                 self._add_ctrl_panel(),
351             ],
352         )
353
354     def _add_plotting_col(self) -> dbc.Col:
355         """Add column with plots and tables. It is placed on the right side.
356         """
357         return dbc.Col(
358             id="col-plotting-area",
359             children=[
360                 dbc.Row(  # Passed / failed tests
361                     id="row-graph-passed",
362                     class_name="g-0 p-2",
363                     children=[
364                         dcc.Loading(children=[
365                             dcc.Graph(
366                                 id="graph-passed",
367                                 figure=self._default_fig_passed
368                             )
369                         ])
370                     ]
371                 ),
372                 dbc.Row(  # Duration
373                     id="row-graph-duration",
374                     class_name="g-0 p-2",
375                     children=[
376                         dcc.Loading(children=[
377                             dcc.Graph(
378                                 id="graph-duration",
379                                 figure=self._default_fig_duration
380                             )
381                         ])
382                     ]
383                 ),
384                 dbc.Row(  # Download
385                     id="row-btn-download",
386                     class_name="g-0 p-2",
387                     children=[
388                         dcc.Loading(children=[
389                             dbc.Button(
390                                 id="btn-download-data",
391                                 children=self._show_tooltip(
392                                     "help-download", "Download"),
393                                 class_name="me-1",
394                                 color="info"
395                             ),
396                             dcc.Download(id="download-data")
397                         ])
398                     ]
399                 )
400             ],
401             width=9,
402         )
403
404     def _add_ctrl_panel(self) -> dbc.Row:
405         """
406         """
407         return dbc.Row(
408             id="row-ctrl-panel",
409             class_name="g-0",
410             children=[
411                 dbc.Row(
412                     class_name="g-0 p-2",
413                     children=[
414                         dbc.Row(
415                             class_name="gy-1",
416                             children=[
417                                 dbc.Label(
418                                     class_name="p-0",
419                                     children=self._show_tooltip(
420                                         "help-dut", "Device under Test")
421                                 ),
422                                 dbc.Row(
423                                     dbc.RadioItems(
424                                         id="ri-duts",
425                                         inline=True,
426                                         value=self.default["dut"],
427                                         options=self.default["duts"]
428                                     )
429                                 )
430                             ]
431                         ),
432                         dbc.Row(
433                             class_name="gy-1",
434                             children=[
435                                 dbc.Label(
436                                     class_name="p-0",
437                                     children=self._show_tooltip(
438                                         "help-ttype", "Test Type"),
439                                 ),
440                                 dbc.RadioItems(
441                                     id="ri-ttypes",
442                                     inline=True,
443                                     value=self.default["ttype"],
444                                     options=self.default["ttypes"]
445                                 )
446                             ]
447                         ),
448                         dbc.Row(
449                             class_name="gy-1",
450                             children=[
451                                 dbc.Label(
452                                     class_name="p-0",
453                                     children=self._show_tooltip(
454                                         "help-cadence", "Cadence"),
455                                 ),
456                                 dbc.RadioItems(
457                                     id="ri-cadences",
458                                     inline=True,
459                                     value=self.default["cadence"],
460                                     options=self.default["cadences"]
461                                 )
462                             ]
463                         ),
464                         dbc.Row(
465                             class_name="gy-1",
466                             children=[
467                                 dbc.Label(
468                                     class_name="p-0",
469                                     children=self._show_tooltip(
470                                         "help-tbed", "Test Bed"),
471                                 ),
472                                 dbc.Select(
473                                     id="dd-tbeds",
474                                     placeholder="Select a test bed...",
475                                     value=self.default["tbed"],
476                                     options=self.default["tbeds"]
477                                 )
478                             ]
479                         ),
480                         dbc.Row(
481                             class_name="gy-1",
482                             children=[
483                                 dbc.Alert(
484                                     id="al-job",
485                                     color="info",
486                                     children=self.default["job"]
487                                 )
488                             ]
489                         ),
490                         dbc.Row(
491                             class_name="g-0 p-2",
492                             children=[
493                                 dbc.Label(
494                                     class_name="gy-1",
495                                     children=self._show_tooltip(
496                                         "help-time-period", "Time Period"),
497                                 ),
498                                 dcc.DatePickerRange(
499                                     id="dpr-period",
500                                     className="d-flex justify-content-center",
501                                     min_date_allowed=\
502                                         datetime.utcnow() - timedelta(
503                                             days=self.time_period),
504                                     max_date_allowed=datetime.utcnow(),
505                                     initial_visible_month=datetime.utcnow(),
506                                     start_date=\
507                                         datetime.utcnow() - timedelta(
508                                             days=self.time_period),
509                                     end_date=datetime.utcnow(),
510                                     display_format="D MMM YY"
511                                 )
512                             ]
513                         )
514                     ]
515                 ),
516             ]
517         )
518
519     class ControlPanel:
520         def __init__(self, panel: dict, default: dict) -> None:
521             self._defaults = {
522                 "ri-ttypes-options": default["ttypes"],
523                 "ri-cadences-options": default["cadences"],
524                 "dd-tbeds-options": default["tbeds"],
525                 "ri-duts-value": default["dut"],
526                 "ri-ttypes-value": default["ttype"],
527                 "ri-cadences-value": default["cadence"],
528                 "dd-tbeds-value": default["tbed"],
529                 "al-job-children": default["job"]
530             }
531             self._panel = deepcopy(self._defaults)
532             if panel:
533                 for key in self._defaults:
534                     self._panel[key] = panel[key]
535
536         def set(self, kwargs: dict) -> None:
537             for key, val in kwargs.items():
538                 if key in self._panel:
539                     self._panel[key] = val
540                 else:
541                     raise KeyError(f"The key {key} is not defined.")
542
543         @property
544         def defaults(self) -> dict:
545             return self._defaults
546
547         @property
548         def panel(self) -> dict:
549             return self._panel
550
551         def get(self, key: str) -> any:
552             return self._panel[key]
553
554         def values(self) -> list:
555             return list(self._panel.values())
556
557     @staticmethod
558     def _generate_options(opts: list) -> list:
559         """
560         """
561         return [{"label": i, "value": i} for i in opts]
562
563     def callbacks(self, app):
564
565         @app.callback(
566             Output("control-panel", "data"),  # Store
567             Output("graph-passed", "figure"),
568             Output("graph-duration", "figure"),
569             Output("ri-ttypes", "options"),
570             Output("ri-cadences", "options"),
571             Output("dd-tbeds", "options"),
572             Output("ri-duts", "value"),
573             Output("ri-ttypes", "value"),
574             Output("ri-cadences", "value"),
575             Output("dd-tbeds", "value"),
576             Output("al-job", "children"),
577             State("control-panel", "data"),  # Store
578             Input("ri-duts", "value"),
579             Input("ri-ttypes", "value"),
580             Input("ri-cadences", "value"),
581             Input("dd-tbeds", "value"),
582             Input("dpr-period", "start_date"),
583             Input("dpr-period", "end_date"),
584             prevent_initial_call=True
585         )
586         def _update_ctrl_panel(cp_data: dict, dut:str, ttype: str, cadence:str,
587                 tbed: str, d_start: str, d_end: str) -> tuple:
588             """
589             """
590
591             ctrl_panel = self.ControlPanel(cp_data, self.default)
592
593             d_start = datetime(int(d_start[0:4]), int(d_start[5:7]),
594                 int(d_start[8:10]))
595             d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10]))
596
597             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
598             if trigger_id == "ri-duts":
599                 ttype_opts = self._generate_options(self._get_ttypes(dut))
600                 ttype_val = ttype_opts[0]["value"]
601                 cad_opts = self._generate_options(
602                     self._get_cadences(dut, ttype_val))
603                 cad_val = cad_opts[0]["value"]
604                 tbed_opts = self._generate_options(
605                     self._get_test_beds(dut, ttype_val, cad_val))
606                 tbed_val = tbed_opts[0]["value"]
607                 ctrl_panel.set({
608                     "ri-duts-value": dut,
609                     "ri-ttypes-options": ttype_opts,
610                     "ri-ttypes-value": ttype_val,
611                     "ri-cadences-options": cad_opts,
612                     "ri-cadences-value": cad_val,
613                     "dd-tbeds-options": tbed_opts,
614                     "dd-tbeds-value": tbed_val
615                 })
616             elif trigger_id == "ri-ttypes":
617                 cad_opts = self._generate_options(
618                     self._get_cadences(ctrl_panel.get("ri-duts-value"), ttype))
619                 cad_val = cad_opts[0]["value"]
620                 tbed_opts = self._generate_options(
621                     self._get_test_beds(ctrl_panel.get("ri-duts-value"),
622                     ttype, cad_val))
623                 tbed_val = tbed_opts[0]["value"]
624                 ctrl_panel.set({
625                     "ri-ttypes-value": ttype,
626                     "ri-cadences-options": cad_opts,
627                     "ri-cadences-value": cad_val,
628                     "dd-tbeds-options": tbed_opts,
629                     "dd-tbeds-value": tbed_val
630                 })
631             elif trigger_id == "ri-cadences":
632                 tbed_opts = self._generate_options(
633                     self._get_test_beds(ctrl_panel.get("ri-duts-value"),
634                     ctrl_panel.get("ri-ttypes-value"), cadence))
635                 tbed_val = tbed_opts[0]["value"]
636                 ctrl_panel.set({
637                     "ri-cadences-value": cadence,
638                     "dd-tbeds-options": tbed_opts,
639                     "dd-tbeds-value": tbed_val
640                 })
641             elif trigger_id == "dd-tbeds":
642                 ctrl_panel.set({
643                     "dd-tbeds-value": tbed
644                 })
645             elif trigger_id == "dpr-period":
646                 pass
647
648             job = self._get_job(
649                 ctrl_panel.get("ri-duts-value"),
650                 ctrl_panel.get("ri-ttypes-value"),
651                 ctrl_panel.get("ri-cadences-value"),
652                 ctrl_panel.get("dd-tbeds-value")
653             )
654             ctrl_panel.set({"al-job-children": job})
655             fig_passed, fig_duration = graph_statistics(
656                 self.data, job, self.layout, d_start, d_end)
657
658             ret_val = [ctrl_panel.panel, fig_passed, fig_duration]
659             ret_val.extend(ctrl_panel.values())
660             return ret_val
661
662         @app.callback(
663             Output("download-data", "data"),
664             State("control-panel", "data"),  # Store
665             State("dpr-period", "start_date"),
666             State("dpr-period", "end_date"),
667             Input("btn-download-data", "n_clicks"),
668             prevent_initial_call=True
669         )
670         def _download_data(cp_data: dict, start: str, end: str, n_clicks: int):
671             """
672             """
673             if not (n_clicks):
674                 raise PreventUpdate
675
676             ctrl_panel = self.ControlPanel(cp_data, self.default)
677
678             job = self._get_job(
679                 ctrl_panel.get("ri-duts-value"),
680                 ctrl_panel.get("ri-ttypes-value"),
681                 ctrl_panel.get("ri-cadences-value"),
682                 ctrl_panel.get("dd-tbeds-value")
683             )
684
685             start = datetime(int(start[0:4]), int(start[5:7]), int(start[8:10]))
686             end = datetime(int(end[0:4]), int(end[5:7]), int(end[8:10]))
687             data = select_data(self.data, job, start, end)
688             data = data.drop(columns=["job", ])
689
690             return dcc.send_data_frame(data.T.to_csv, f"{job}-stats.csv")
691
692         @app.callback(
693             Output("row-metadata", "children"),
694             Output("offcanvas-metadata", "is_open"),
695             Input("graph-passed", "clickData"),
696             Input("graph-duration", "clickData"),
697             prevent_initial_call=True
698         )
699         def _show_metadata_from_graphs(
700                 passed_data: dict, duration_data: dict) -> tuple:
701             """
702             """
703
704             if not (passed_data or duration_data):
705                 raise PreventUpdate
706
707             metadata = no_update
708             open_canvas = False
709             title = "Job Statistics"
710             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
711             if trigger_id == "graph-passed":
712                 graph_data = passed_data["points"][0].get("hovertext", "")
713             elif trigger_id == "graph-duration":
714                 graph_data = duration_data["points"][0].get("text", "")
715             if graph_data:
716                 metadata = [
717                     dbc.Card(
718                         class_name="gy-2 p-0",
719                         children=[
720                             dbc.CardHeader(children=[
721                                 dcc.Clipboard(
722                                     target_id="metadata",
723                                     title="Copy",
724                                     style={"display": "inline-block"}
725                                 ),
726                                 title
727                             ]),
728                             dbc.CardBody(
729                                 id="metadata",
730                                 class_name="p-0",
731                                 children=[dbc.ListGroup(
732                                     children=[
733                                         dbc.ListGroupItem(
734                                             [
735                                                 dbc.Badge(
736                                                     x.split(":")[0]
737                                                 ),
738                                                 x.split(": ")[1]
739                                             ]
740                                         ) for x in graph_data.split("<br>")
741                                     ],
742                                     flush=True),
743                                 ]
744                             )
745                         ]
746                     )
747                 ]
748                 open_canvas = True
749
750             return metadata, open_canvas