1 # Copyright (c) 2022 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Plotly Dash HTML layout override.
19 import dash_bootstrap_components as dbc
21 from flask import Flask
24 from dash import callback_context
25 from dash import Input, Output, State
26 from yaml import load, FullLoader, YAMLError
27 from copy import deepcopy
29 from ..data.data import Data
30 from .tables import table_failed
34 """The layout of the dash app and the callbacks.
37 # The default job displayed when the page is loaded first time.
38 DEFAULT_JOB = "csit-vpp-perf-mrr-daily-master-2n-icx"
40 def __init__(self, app: Flask, html_layout_file: str, data_spec_file: str,
41 tooltip_file: str) -> None:
43 - save the input parameters,
44 - read and pre-process the data,
45 - prepare data fro the control panel,
46 - read HTML layout file,
47 - read tooltips from the tooltip file.
49 :param app: Flask application running the dash application.
50 :param html_layout_file: Path and name of the file specifying the HTML
51 layout of the dash application.
52 :param data_spec_file: Path and name of the file specifying the data to
53 be read from parquets for this application.
54 :param tooltip_file: Path and name of the yaml file specifying the
57 :type html_layout_file: str
58 :type data_spec_file: str
59 :type tooltip_file: str
64 self._html_layout_file = html_layout_file
65 self._data_spec_file = data_spec_file
66 self._tooltip_file = tooltip_file
69 data_stats, data_mrr, data_ndrpdr = Data(
70 data_spec_file=self._data_spec_file,
72 ).read_stats(days=10) # To be sure
74 df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
76 # Prepare information for the control panel:
77 jobs = sorted(list(df_tst_info["job"].unique()))
86 lst_job = job.split("-")
87 job_info["job"].append(job)
88 job_info["dut"].append(lst_job[1])
89 job_info["ttype"].append(lst_job[3])
90 job_info["cadence"].append(lst_job[4])
91 job_info["tbed"].append("-".join(lst_job[-2:]))
92 self.df_job_info = pd.DataFrame.from_dict(job_info)
94 self._default = self._set_job_params(self.DEFAULT_JOB)
96 # Pre-process the data:
102 "dut_version": list(),
107 df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
108 last_build = max(df_job["build"].unique())
109 df_build = df_job.loc[(df_job["build"] == last_build)]
110 tst_info["job"].append(job)
111 tst_info["build"].append(last_build)
112 tst_info["start"].append(data_stats.loc[
113 (data_stats["job"] == job) &
114 (data_stats["build"] == last_build)
115 ]["start_time"].iloc[-1].strftime('%Y-%m-%d %H:%M'))
116 tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
117 tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
118 tst_info["hosts"].append(df_build["hosts"].iloc[-1])
119 failed_tests = df_build.loc[(df_build["passed"] == False)]\
120 ["test_id"].to_list()
123 for tst in failed_tests:
124 lst_tst = tst.split(".")
125 suite = lst_tst[-2].replace("2n1l-", "").\
126 replace("1n1l-", "").replace("2n-", "")
127 l_failed.append(f"{suite.split('-')[0]}-{lst_tst[-1]}")
130 tst_info["lst_failed"].append(sorted(l_failed))
132 self._data = pd.DataFrame.from_dict(tst_info)
135 self._html_layout = str()
136 self._tooltips = dict()
139 with open(self._html_layout_file, "r") as file_read:
140 self._html_layout = file_read.read()
141 except IOError as err:
143 f"Not possible to open the file {self._html_layout_file}\n{err}"
147 with open(self._tooltip_file, "r") as file_read:
148 self._tooltips = load(file_read, Loader=FullLoader)
149 except IOError as err:
151 f"Not possible to open the file {self._tooltip_file}\n{err}"
153 except YAMLError as err:
155 f"An error occurred while parsing the specification file "
156 f"{self._tooltip_file}\n{err}"
159 self._default_tab_failed = table_failed(self.data, self._default["job"])
162 if self._app is not None and hasattr(self, 'callbacks'):
163 self.callbacks(self._app)
166 def html_layout(self) -> dict:
167 return self._html_layout
170 def data(self) -> pd.DataFrame:
174 def default(self) -> dict:
177 def _get_duts(self) -> list:
178 """Get the list of DUTs from the pre-processed information about jobs.
180 :returns: Alphabeticaly sorted list of DUTs.
183 return sorted(list(self.df_job_info["dut"].unique()))
185 def _get_ttypes(self, dut: str) -> list:
186 """Get the list of test types from the pre-processed information about
189 :param dut: The DUT for which the list of test types will be populated.
191 :returns: Alphabeticaly sorted list of test types.
194 return sorted(list(self.df_job_info.loc[(
195 self.df_job_info["dut"] == dut
196 )]["ttype"].unique()))
198 def _get_cadences(self, dut: str, ttype: str) -> list:
199 """Get the list of cadences from the pre-processed information about
202 :param dut: The DUT for which the list of cadences will be populated.
203 :param ttype: The test type for which the list of cadences will be
207 :returns: Alphabeticaly sorted list of cadences.
210 return sorted(list(self.df_job_info.loc[(
211 (self.df_job_info["dut"] == dut) &
212 (self.df_job_info["ttype"] == ttype)
213 )]["cadence"].unique()))
215 def _get_test_beds(self, dut: str, ttype: str, cadence: str) -> list:
216 """Get the list of test beds from the pre-processed information about
219 :param dut: The DUT for which the list of test beds will be populated.
220 :param ttype: The test type for which the list of test beds will be
222 :param cadence: The cadence for which the list of test beds will be
227 :returns: Alphabeticaly sorted list of test beds.
230 return sorted(list(self.df_job_info.loc[(
231 (self.df_job_info["dut"] == dut) &
232 (self.df_job_info["ttype"] == ttype) &
233 (self.df_job_info["cadence"] == cadence)
234 )]["tbed"].unique()))
236 def _get_job(self, dut, ttype, cadence, testbed):
237 """Get the name of a job defined by dut, ttype, cadence, test bed.
238 Input information comes from the control panel.
240 :param dut: The DUT for which the job name will be created.
241 :param ttype: The test type for which the job name will be created.
242 :param cadence: The cadence for which the job name will be created.
243 :param testbed: The test bed for which the job name will be created.
251 return self.df_job_info.loc[(
252 (self.df_job_info["dut"] == dut) &
253 (self.df_job_info["ttype"] == ttype) &
254 (self.df_job_info["cadence"] == cadence) &
255 (self.df_job_info["tbed"] == testbed)
259 def _generate_options(opts: list) -> list:
260 """Return list of options for radio items in control panel. The items in
261 the list are dictionaries with keys "label" and "value".
263 :params opts: List of options (str) to be used for the generated list.
265 :returns: List of options (dict).
268 return [{"label": i, "value": i} for i in opts]
270 def _set_job_params(self, job: str) -> dict:
271 """Create a dictionary with all options and values for (and from) the
274 :params job: The name of job for and from which the dictionary will be
277 :returns: Dictionary with all options and values for (and from) the
282 lst_job = job.split("-")
287 "cadence": lst_job[4],
288 "tbed": "-".join(lst_job[-2:]),
289 "duts": self._generate_options(self._get_duts()),
290 "ttypes": self._generate_options(self._get_ttypes(lst_job[1])),
291 "cadences": self._generate_options(self._get_cadences(
292 lst_job[1], lst_job[3])),
293 "tbeds": self._generate_options(self._get_test_beds(
294 lst_job[1], lst_job[3], lst_job[4]))
297 def _show_tooltip(self, id: str, title: str,
298 clipboard_id: str=None) -> list:
299 """Generate list of elements to display a text (e.g. a title) with a
300 tooltip and optionaly with Copy&Paste icon and the clipboard
301 functionality enabled.
303 :param id: Tooltip ID.
304 :param title: A text for which the tooltip will be displayed.
305 :param clipboard_id: If defined, a Copy&Paste icon is displayed and the
306 clipboard functionality is enabled.
309 :type clipboard_id: str
310 :returns: List of elements to display a text with a tooltip and
311 optionaly with Copy&Paste icon.
316 dcc.Clipboard(target_id=clipboard_id, title="Copy URL") \
317 if clipboard_id else str(),
325 class_name="border ms-1",
328 children=self._tooltips.get(id, str()),
334 def add_content(self):
335 """Top level method which generated the web page.
338 - Store for user input data,
340 - Main area with control panel and ploting area.
342 If no HTML layout is provided, an error message is displayed instead.
344 :returns: The HTML div with teh whole page.
352 dcc.Store(id="control-panel"),
364 self._add_ctrl_col(),
365 self._add_plotting_col(),
383 def _add_navbar(self):
384 """Add nav element with navigation panel. It is placed on the top.
386 :returns: Navigation bar.
387 :rtype: dbc.NavbarSimple
390 return dbc.NavbarSimple(
391 id="navbarsimple-main",
395 "Continuous Performance News",
404 brand_external_link=True,
409 def _add_ctrl_col(self) -> dbc.Col:
410 """Add column with control panel. It is placed on the left side.
412 :returns: Column with the control panel.
419 self._add_ctrl_panel(),
423 def _add_plotting_col(self) -> dbc.Col:
424 """Add column with tables. It is placed on the right side.
426 :returns: Column with tables.
431 id="col-plotting-area",
433 dbc.Row( # Failed tests
434 id="row-table-failed",
435 class_name="g-0 p-2",
436 children=self._default_tab_failed
442 def _add_ctrl_panel(self) -> dbc.Row:
443 """Add control panel.
445 :returns: Control panel.
453 class_name="g-0 p-2",
460 children=self._show_tooltip(
461 "help-dut", "Device under Test")
467 value=self.default["dut"],
468 options=self.default["duts"]
478 children=self._show_tooltip(
479 "help-ttype", "Test Type"),
484 value=self.default["ttype"],
485 options=self.default["ttypes"]
494 children=self._show_tooltip(
495 "help-cadence", "Cadence"),
500 value=self.default["cadence"],
501 options=self.default["cadences"]
510 children=self._show_tooltip(
511 "help-tbed", "Test Bed"),
515 placeholder="Select a test bed...",
516 value=self.default["tbed"],
517 options=self.default["tbeds"]
527 children=self.default["job"]
540 def __init__(self, panel: dict, default: dict) -> None:
545 "ri-ttypes-options": default["ttypes"],
546 "ri-cadences-options": default["cadences"],
547 "dd-tbeds-options": default["tbeds"],
548 "ri-duts-value": default["dut"],
549 "ri-ttypes-value": default["ttype"],
550 "ri-cadences-value": default["cadence"],
551 "dd-tbeds-value": default["tbed"],
552 "al-job-children": default["job"]
554 self._panel = deepcopy(self._defaults)
556 for key in self._defaults:
557 self._panel[key] = panel[key]
559 def set(self, kwargs: dict) -> None:
560 for key, val in kwargs.items():
561 if key in self._panel:
562 self._panel[key] = val
564 raise KeyError(f"The key {key} is not defined.")
567 def defaults(self) -> dict:
568 return self._defaults
571 def panel(self) -> dict:
574 def get(self, key: str) -> any:
575 return self._panel[key]
577 def values(self) -> list:
578 return list(self._panel.values())
580 def callbacks(self, app):
583 Output("control-panel", "data"), # Store
584 Output("row-table-failed", "children"),
585 Output("ri-ttypes", "options"),
586 Output("ri-cadences", "options"),
587 Output("dd-tbeds", "options"),
588 Output("ri-duts", "value"),
589 Output("ri-ttypes", "value"),
590 Output("ri-cadences", "value"),
591 Output("dd-tbeds", "value"),
592 Output("al-job", "children"),
593 State("control-panel", "data"), # Store
594 Input("ri-duts", "value"),
595 Input("ri-ttypes", "value"),
596 Input("ri-cadences", "value"),
597 Input("dd-tbeds", "value"),
599 def _update_ctrl_panel(cp_data: dict, dut:str, ttype: str, cadence:str,
604 ctrl_panel = self.ControlPanel(cp_data, self.default)
606 trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
607 if trigger_id == "ri-duts":
608 ttype_opts = self._generate_options(self._get_ttypes(dut))
609 ttype_val = ttype_opts[0]["value"]
610 cad_opts = self._generate_options(
611 self._get_cadences(dut, ttype_val))
612 cad_val = cad_opts[0]["value"]
613 tbed_opts = self._generate_options(
614 self._get_test_beds(dut, ttype_val, cad_val))
615 tbed_val = tbed_opts[0]["value"]
617 "ri-duts-value": dut,
618 "ri-ttypes-options": ttype_opts,
619 "ri-ttypes-value": ttype_val,
620 "ri-cadences-options": cad_opts,
621 "ri-cadences-value": cad_val,
622 "dd-tbeds-options": tbed_opts,
623 "dd-tbeds-value": tbed_val
625 elif trigger_id == "ri-ttypes":
626 cad_opts = self._generate_options(
627 self._get_cadences(ctrl_panel.get("ri-duts-value"), ttype))
628 cad_val = cad_opts[0]["value"]
629 tbed_opts = self._generate_options(
630 self._get_test_beds(ctrl_panel.get("ri-duts-value"),
632 tbed_val = tbed_opts[0]["value"]
634 "ri-ttypes-value": ttype,
635 "ri-cadences-options": cad_opts,
636 "ri-cadences-value": cad_val,
637 "dd-tbeds-options": tbed_opts,
638 "dd-tbeds-value": tbed_val
640 elif trigger_id == "ri-cadences":
641 tbed_opts = self._generate_options(
642 self._get_test_beds(ctrl_panel.get("ri-duts-value"),
643 ctrl_panel.get("ri-ttypes-value"), cadence))
644 tbed_val = tbed_opts[0]["value"]
646 "ri-cadences-value": cadence,
647 "dd-tbeds-options": tbed_opts,
648 "dd-tbeds-value": tbed_val
650 elif trigger_id == "dd-tbeds":
652 "dd-tbeds-value": tbed
656 ctrl_panel.get("ri-duts-value"),
657 ctrl_panel.get("ri-ttypes-value"),
658 ctrl_panel.get("ri-cadences-value"),
659 ctrl_panel.get("dd-tbeds-value")
661 ctrl_panel.set({"al-job-children": job})
662 tab_failed = table_failed(self.data, job)
668 ret_val.extend(ctrl_panel.values())