from copy import deepcopy
from ..data.data import Data
-from .tables import table_failed
+from ..utils.constants import Constants as C
+from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url
+from ..utils.url_processing import url_decode
+from ..data.data import Data
+from .tables import table_news
class Layout:
+ """The layout of the dash app and the callbacks.
"""
- """
-
- DEFAULT_JOB = "csit-vpp-perf-mrr-daily-master-2n-icx"
- URL_STYLE = {
- "background-color": "#d2ebf5",
- "border-color": "#bce1f1",
- "color": "#135d7c"
- }
-
- def __init__(self, app: Flask, html_layout_file: str,
- data_spec_file: str, tooltip_file: str) -> None:
- """
+ def __init__(self, app: Flask, html_layout_file: str, data_spec_file: str,
+ tooltip_file: str) -> None:
+ """Initialization:
+ - save the input parameters,
+ - read and pre-process the data,
+ - prepare data fro the control panel,
+ - read HTML layout file,
+ - read tooltips from the tooltip file.
+
+ :param app: Flask application running the dash application.
+ :param html_layout_file: Path and name of the file specifying the HTML
+ layout of the dash application.
+ :param data_spec_file: Path and name of the file specifying the data to
+ be read from parquets for this application.
+ :param tooltip_file: Path and name of the yaml file specifying the
+ tooltips.
+ :type app: Flask
+ :type html_layout_file: str
+ :type data_spec_file: str
+ :type tooltip_file: str
"""
# Inputs
data_stats, data_mrr, data_ndrpdr = Data(
data_spec_file=self._data_spec_file,
debug=True
- ).read_stats(days=10) # To be sure
+ ).read_stats(days=C.NEWS_TIME_PERIOD)
df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
+ # Prepare information for the control panel:
jobs = sorted(list(df_tst_info["job"].unique()))
job_info = {
"job": list(),
job_info["tbed"].append("-".join(lst_job[-2:]))
self.df_job_info = pd.DataFrame.from_dict(job_info)
- self._default = self._set_job_params(self.DEFAULT_JOB)
+ self._default = self._set_job_params(C.NEWS_DEFAULT_JOB)
+
+ # Pre-process the data:
+
+ def _create_test_name(test: str) -> str:
+ lst_tst = test.split(".")
+ suite = lst_tst[-2].replace("2n1l-", "").replace("1n1l-", "").\
+ replace("2n-", "")
+ return f"{suite.split('-')[0]}-{lst_tst[-1]}"
+
+ def _get_rindex(array: list, itm: any) -> int:
+ return len(array) - 1 - array[::-1].index(itm)
tst_info = {
"job": list(),
"dut_type": list(),
"dut_version": list(),
"hosts": list(),
- "lst_failed": list()
+ "failed": list(),
+ "regressions": list(),
+ "progressions": list()
}
for job in jobs:
+ # Create lists of failed tests:
df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
last_build = max(df_job["build"].unique())
df_build = df_job.loc[(df_job["build"] == last_build)]
l_failed = list()
try:
for tst in failed_tests:
- lst_tst = tst.split(".")
- suite = lst_tst[-2].replace("2n1l-", "").\
- replace("1n1l-", "").replace("2n-", "")
- l_failed.append(f"{suite.split('-')[0]}-{lst_tst[-1]}")
+ l_failed.append(_create_test_name(tst))
except KeyError:
l_failed = list()
- tst_info["lst_failed"].append(sorted(l_failed))
+ tst_info["failed"].append(sorted(l_failed))
+
+ # Create lists of regressions and progressions:
+ l_reg = list()
+ l_prog = list()
+
+ tests = df_job["test_id"].unique()
+ for test in tests:
+ tst_data = df_job.loc[df_job["test_id"] == test].sort_values(
+ by="start_time", ignore_index=True)
+ x_axis = tst_data["start_time"].tolist()
+ if "-ndrpdr" in test:
+ tst_data = tst_data.dropna(
+ subset=["result_pdr_lower_rate_value", ]
+ )
+ if tst_data.empty:
+ continue
+ try:
+ anomalies, _, _ = classify_anomalies({
+ k: v for k, v in zip(
+ x_axis,
+ tst_data["result_ndr_lower_rate_value"].tolist()
+ )
+ })
+ except ValueError:
+ continue
+ if "progression" in anomalies:
+ l_prog.append((
+ _create_test_name(test).replace("-ndrpdr", "-ndr"),
+ x_axis[_get_rindex(anomalies, "progression")]
+ ))
+ if "regression" in anomalies:
+ l_reg.append((
+ _create_test_name(test).replace("-ndrpdr", "-ndr"),
+ x_axis[_get_rindex(anomalies, "regression")]
+ ))
+ try:
+ anomalies, _, _ = classify_anomalies({
+ k: v for k, v in zip(
+ x_axis,
+ tst_data["result_pdr_lower_rate_value"].tolist()
+ )
+ })
+ except ValueError:
+ continue
+ if "progression" in anomalies:
+ l_prog.append((
+ _create_test_name(test).replace("-ndrpdr", "-pdr"),
+ x_axis[_get_rindex(anomalies, "progression")]
+ ))
+ if "regression" in anomalies:
+ l_reg.append((
+ _create_test_name(test).replace("-ndrpdr", "-pdr"),
+ x_axis[_get_rindex(anomalies, "regression")]
+ ))
+ else: # mrr
+ tst_data = tst_data.dropna(
+ subset=["result_receive_rate_rate_avg", ]
+ )
+ if tst_data.empty:
+ continue
+ try:
+ anomalies, _, _ = classify_anomalies({
+ k: v for k, v in zip(
+ x_axis,
+ tst_data["result_receive_rate_rate_avg"].\
+ tolist()
+ )
+ })
+ except ValueError:
+ continue
+ if "progression" in anomalies:
+ l_prog.append((
+ _create_test_name(test),
+ x_axis[_get_rindex(anomalies, "progression")]
+ ))
+ if "regression" in anomalies:
+ l_reg.append((
+ _create_test_name(test),
+ x_axis[_get_rindex(anomalies, "regression")]
+ ))
+
+ tst_info["regressions"].append(
+ sorted(l_reg, key=lambda k: k[1], reverse=True))
+ tst_info["progressions"].append(
+ sorted(l_prog, key=lambda k: k[1], reverse=True))
self._data = pd.DataFrame.from_dict(tst_info)
# Read from files:
- self._html_layout = ""
+ self._html_layout = str()
self._tooltips = dict()
try:
f"{self._tooltip_file}\n{err}"
)
- self._default_tab_failed = table_failed(self.data, self._default["job"])
+ self._default_tab_failed = table_news(self.data, self._default["job"])
# Callbacks:
if self._app is not None and hasattr(self, 'callbacks'):
return self._data
@property
- def default(self) -> any:
+ def default(self) -> dict:
return self._default
def _get_duts(self) -> list:
- """
+ """Get the list of DUTs from the pre-processed information about jobs.
+
+ :returns: Alphabeticaly sorted list of DUTs.
+ :rtype: list
"""
return sorted(list(self.df_job_info["dut"].unique()))
def _get_ttypes(self, dut: str) -> list:
- """
+ """Get the list of test types from the pre-processed information about
+ jobs.
+
+ :param dut: The DUT for which the list of test types will be populated.
+ :type dut: str
+ :returns: Alphabeticaly sorted list of test types.
+ :rtype: list
"""
return sorted(list(self.df_job_info.loc[(
self.df_job_info["dut"] == dut
)]["ttype"].unique()))
def _get_cadences(self, dut: str, ttype: str) -> list:
- """
+ """Get the list of cadences from the pre-processed information about
+ jobs.
+
+ :param dut: The DUT for which the list of cadences will be populated.
+ :param ttype: The test type for which the list of cadences will be
+ populated.
+ :type dut: str
+ :type ttype: str
+ :returns: Alphabeticaly sorted list of cadences.
+ :rtype: list
"""
return sorted(list(self.df_job_info.loc[(
(self.df_job_info["dut"] == dut) &
)]["cadence"].unique()))
def _get_test_beds(self, dut: str, ttype: str, cadence: str) -> list:
- """
+ """Get the list of test beds from the pre-processed information about
+ jobs.
+
+ :param dut: The DUT for which the list of test beds will be populated.
+ :param ttype: The test type for which the list of test beds will be
+ populated.
+ :param cadence: The cadence for which the list of test beds will be
+ populated.
+ :type dut: str
+ :type ttype: str
+ :type cadence: str
+ :returns: Alphabeticaly sorted list of test beds.
+ :rtype: list
"""
return sorted(list(self.df_job_info.loc[(
(self.df_job_info["dut"] == dut) &
)]["tbed"].unique()))
def _get_job(self, dut, ttype, cadence, testbed):
- """Get the name of a job defined by dut, ttype, cadence, testbed.
-
- Input information comes from control panel.
+ """Get the name of a job defined by dut, ttype, cadence, test bed.
+ Input information comes from the control panel.
+
+ :param dut: The DUT for which the job name will be created.
+ :param ttype: The test type for which the job name will be created.
+ :param cadence: The cadence for which the job name will be created.
+ :param testbed: The test bed for which the job name will be created.
+ :type dut: str
+ :type ttype: str
+ :type cadence: str
+ :type testbed: str
+ :returns: Job name.
+ :rtype: str
"""
return self.df_job_info.loc[(
(self.df_job_info["dut"] == dut) &
(self.df_job_info["tbed"] == testbed)
)]["job"].item()
- def _set_job_params(self, job: str) -> dict:
+ @staticmethod
+ def _generate_options(opts: list) -> list:
+ """Return list of options for radio items in control panel. The items in
+ the list are dictionaries with keys "label" and "value".
+
+ :params opts: List of options (str) to be used for the generated list.
+ :type opts: list
+ :returns: List of options (dict).
+ :rtype: list
"""
+ return [{"label": i, "value": i} for i in opts]
+
+ def _set_job_params(self, job: str) -> dict:
+ """Create a dictionary with all options and values for (and from) the
+ given job.
+
+ :params job: The name of job for and from which the dictionary will be
+ created.
+ :type job: str
+ :returns: Dictionary with all options and values for (and from) the
+ given job.
+ :rtype: dict
"""
+
lst_job = job.split("-")
return {
"job": job,
lst_job[1], lst_job[3], lst_job[4]))
}
- def _show_tooltip(self, id: str, title: str,
- clipboard_id: str=None) -> list:
- """
- """
- return [
- dcc.Clipboard(target_id=clipboard_id, title="Copy URL") \
- if clipboard_id else str(),
- f"{title} ",
- dbc.Badge(
- id=id,
- children="?",
- pill=True,
- color="white",
- text_color="info",
- class_name="border ms-1",
- ),
- dbc.Tooltip(
- children=self._tooltips.get(id, str()),
- target=id,
- placement="auto"
- )
- ]
-
def add_content(self):
+ """Top level method which generated the web page.
+
+ It generates:
+ - Store for user input data,
+ - Navigation bar,
+ - Main area with control panel and ploting area.
+
+ If no HTML layout is provided, an error message is displayed instead.
+
+ :returns: The HTML div with teh whole page.
+ :rtype: html.Div
"""
- """
+
if self.html_layout:
return html.Div(
id="div-main",
children=[
dcc.Store(id="control-panel"),
+ dcc.Location(id="url", refresh=False),
dbc.Row(
id="row-navbar",
class_name="g-0",
def _add_navbar(self):
"""Add nav element with navigation panel. It is placed on the top.
+
+ :returns: Navigation bar.
+ :rtype: dbc.NavbarSimple
"""
+
return dbc.NavbarSimple(
id="navbarsimple-main",
children=[
)
def _add_ctrl_col(self) -> dbc.Col:
- """Add column with controls. It is placed on the left side.
+ """Add column with control panel. It is placed on the left side.
+
+ :returns: Column with the control panel.
+ :rtype: dbc.col
"""
+
return dbc.Col(
id="col-controls",
children=[
)
def _add_plotting_col(self) -> dbc.Col:
- """Add column with plots and tables. It is placed on the right side.
+ """Add column with tables. It is placed on the right side.
+
+ :returns: Column with tables.
+ :rtype: dbc.col
"""
+
return dbc.Col(
id="col-plotting-area",
children=[
id="row-table-failed",
class_name="g-0 p-2",
children=self._default_tab_failed
+ ),
+ dbc.Row(
+ class_name="g-0 p-2",
+ align="center",
+ justify="start",
+ children=[
+ dbc.InputGroup(
+ class_name="me-1",
+ children=[
+ dbc.InputGroupText(
+ style=C.URL_STYLE,
+ children=show_tooltip(
+ self._tooltips,
+ "help-url", "URL",
+ "input-url"
+ )
+ ),
+ dbc.Input(
+ id="input-url",
+ readonly=True,
+ type="url",
+ style=C.URL_STYLE,
+ value=""
+ )
+ ]
+ )
+ ]
)
],
width=9,
)
def _add_ctrl_panel(self) -> dbc.Row:
- """
+ """Add control panel.
+
+ :returns: Control panel.
+ :rtype: dbc.Row
"""
return dbc.Row(
id="row-ctrl-panel",
children=[
dbc.Label(
class_name="p-0",
- children=self._show_tooltip(
+ children=show_tooltip(self._tooltips,
"help-dut", "Device under Test")
),
dbc.Row(
children=[
dbc.Label(
class_name="p-0",
- children=self._show_tooltip(
+ children=show_tooltip(self._tooltips,
"help-ttype", "Test Type"),
),
dbc.RadioItems(
children=[
dbc.Label(
class_name="p-0",
- children=self._show_tooltip(
+ children=show_tooltip(self._tooltips,
"help-cadence", "Cadence"),
),
dbc.RadioItems(
children=[
dbc.Label(
class_name="p-0",
- children=self._show_tooltip(
+ children=show_tooltip(self._tooltips,
"help-tbed", "Test Bed"),
),
dbc.Select(
)
class ControlPanel:
+ """
+ """
+
def __init__(self, panel: dict, default: dict) -> None:
+ """
+ """
+
self._defaults = {
"ri-ttypes-options": default["ttypes"],
"ri-cadences-options": default["cadences"],
def values(self) -> list:
return list(self._panel.values())
- @staticmethod
- def _generate_options(opts: list) -> list:
- return [{"label": i, "value": i} for i in opts]
-
def callbacks(self, app):
@app.callback(
Output("control-panel", "data"), # Store
Output("row-table-failed", "children"),
+ Output("input-url", "value"),
Output("ri-ttypes", "options"),
Output("ri-cadences", "options"),
Output("dd-tbeds", "options"),
Input("ri-ttypes", "value"),
Input("ri-cadences", "value"),
Input("dd-tbeds", "value"),
+ Input("url", "href")
)
def _update_ctrl_panel(cp_data: dict, dut:str, ttype: str, cadence:str,
- tbed: str) -> tuple:
+ tbed: str, href: str) -> tuple:
"""
"""
ctrl_panel = self.ControlPanel(cp_data, self.default)
+ # Parse the url:
+ parsed_url = url_decode(href)
+ if parsed_url:
+ url_params = parsed_url["params"]
+ else:
+ url_params = None
+
trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
if trigger_id == "ri-duts":
ttype_opts = self._generate_options(self._get_ttypes(dut))
ctrl_panel.set({
"dd-tbeds-value": tbed
})
+ elif trigger_id == "url":
+ # TODO: Add verification
+ if url_params:
+ new_job = url_params.get("job", list())[0]
+ if new_job:
+ job_params = self._set_job_params(new_job)
+ ctrl_panel = self.ControlPanel(None, job_params)
+ else:
+ ctrl_panel = self.ControlPanel(cp_data, self.default)
+ job = self._get_job(
+ ctrl_panel.get("ri-duts-value"),
+ ctrl_panel.get("ri-ttypes-value"),
+ ctrl_panel.get("ri-cadences-value"),
+ ctrl_panel.get("dd-tbeds-value")
+ )
job = self._get_job(
ctrl_panel.get("ri-duts-value"),
ctrl_panel.get("dd-tbeds-value")
)
ctrl_panel.set({"al-job-children": job})
- tab_failed = table_failed(self.data, job)
+ tab_failed = table_news(self.data, job)
ret_val = [
ctrl_panel.panel,
- tab_failed
+ tab_failed,
+ gen_new_url(parsed_url, {"job": job})
]
ret_val.extend(ctrl_panel.values())
return ret_val
Code Review / csit.git / blobdiff