--- /dev/null
+# Copyright (c) 2022 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Plotly Dash HTML layout override.
+"""
+
+import logging
+import pandas as pd
+import dash_bootstrap_components as dbc
+
+from flask import Flask
+from dash import dcc
+from dash import html
+from dash import callback_context
+from dash import Input, Output
+from yaml import load, FullLoader, YAMLError
+
+from ..data.data import Data
+from ..utils.constants import Constants as C
+from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url
+from ..utils.url_processing import url_decode
+from ..data.data import Data
+from .tables import table_summary
+
+
+class Layout:
+ """The layout of the dash app and the callbacks.
+ """
+
+ def __init__(self, app: Flask, html_layout_file: str, data_spec_file: str,
+ tooltip_file: str) -> None:
+ """Initialization:
+ - save the input parameters,
+ - read and pre-process the data,
+ - prepare data for the control panel,
+ - read HTML layout file,
+ - read tooltips from the tooltip file.
+
+ :param app: Flask application running the dash application.
+ :param html_layout_file: Path and name of the file specifying the HTML
+ layout of the dash application.
+ :param data_spec_file: Path and name of the file specifying the data to
+ be read from parquets for this application.
+ :param tooltip_file: Path and name of the yaml file specifying the
+ tooltips.
+ :type app: Flask
+ :type html_layout_file: str
+ :type data_spec_file: str
+ :type tooltip_file: str
+ """
+
+ # Inputs
+ self._app = app
+ self._html_layout_file = html_layout_file
+ self._data_spec_file = data_spec_file
+ self._tooltip_file = tooltip_file
+
+ # Read the data:
+ data_stats, data_mrr, data_ndrpdr = Data(
+ data_spec_file=self._data_spec_file,
+ debug=True
+ ).read_stats(days=C.NEWS_TIME_PERIOD)
+
+ df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
+
+ # Prepare information for the control panel:
+ self._jobs = sorted(list(df_tst_info["job"].unique()))
+ d_job_info = {
+ "job": list(),
+ "dut": list(),
+ "ttype": list(),
+ "cadence": list(),
+ "tbed": list()
+ }
+ for job in self._jobs:
+ lst_job = job.split("-")
+ d_job_info["job"].append(job)
+ d_job_info["dut"].append(lst_job[1])
+ d_job_info["ttype"].append(lst_job[3])
+ d_job_info["cadence"].append(lst_job[4])
+ d_job_info["tbed"].append("-".join(lst_job[-2:]))
+ self.job_info = pd.DataFrame.from_dict(d_job_info)
+
+ # Pre-process the data:
+
+ def _create_test_name(test: str) -> str:
+ lst_tst = test.split(".")
+ suite = lst_tst[-2].replace("2n1l-", "").replace("1n1l-", "").\
+ replace("2n-", "")
+ return f"{suite.split('-')[0]}-{lst_tst[-1]}"
+
+ def _get_rindex(array: list, itm: any) -> int:
+ return len(array) - 1 - array[::-1].index(itm)
+
+ tst_info = {
+ "job": list(),
+ "build": list(),
+ "start": list(),
+ "dut_type": list(),
+ "dut_version": list(),
+ "hosts": list(),
+ "failed": list(),
+ "regressions": list(),
+ "progressions": list()
+ }
+ for job in self._jobs:
+ # Create lists of failed tests:
+ df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
+ last_build = str(max(pd.to_numeric(df_job["build"].unique())))
+ df_build = df_job.loc[(df_job["build"] == last_build)]
+ tst_info["job"].append(job)
+ tst_info["build"].append(last_build)
+ tst_info["start"].append(data_stats.loc[
+ (data_stats["job"] == job) &
+ (data_stats["build"] == last_build)
+ ]["start_time"].iloc[-1].strftime('%Y-%m-%d %H:%M'))
+ tst_info["dut_type"].append(df_build["dut_type"].iloc[-1])
+ tst_info["dut_version"].append(df_build["dut_version"].iloc[-1])
+ tst_info["hosts"].append(df_build["hosts"].iloc[-1])
+ failed_tests = df_build.loc[(df_build["passed"] == False)]\
+ ["test_id"].to_list()
+ l_failed = list()
+ try:
+ for tst in failed_tests:
+ l_failed.append(_create_test_name(tst))
+ except KeyError:
+ l_failed = list()
+ tst_info["failed"].append(sorted(l_failed))
+
+ # Create lists of regressions and progressions:
+ l_reg = list()
+ l_prog = list()
+
+ tests = df_job["test_id"].unique()
+ for test in tests:
+ tst_data = df_job.loc[df_job["test_id"] == test].sort_values(
+ by="start_time", ignore_index=True)
+ x_axis = tst_data["start_time"].tolist()
+ if "-ndrpdr" in test:
+ tst_data = tst_data.dropna(
+ subset=["result_pdr_lower_rate_value", ]
+ )
+ if tst_data.empty:
+ continue
+ try:
+ anomalies, _, _ = classify_anomalies({
+ k: v for k, v in zip(
+ x_axis,
+ tst_data["result_ndr_lower_rate_value"].tolist()
+ )
+ })
+ except ValueError:
+ continue
+ if "progression" in anomalies:
+ l_prog.append((
+ _create_test_name(test).replace("-ndrpdr", "-ndr"),
+ x_axis[_get_rindex(anomalies, "progression")]
+ ))
+ if "regression" in anomalies:
+ l_reg.append((
+ _create_test_name(test).replace("-ndrpdr", "-ndr"),
+ x_axis[_get_rindex(anomalies, "regression")]
+ ))
+ try:
+ anomalies, _, _ = classify_anomalies({
+ k: v for k, v in zip(
+ x_axis,
+ tst_data["result_pdr_lower_rate_value"].tolist()
+ )
+ })
+ except ValueError:
+ continue
+ if "progression" in anomalies:
+ l_prog.append((
+ _create_test_name(test).replace("-ndrpdr", "-pdr"),
+ x_axis[_get_rindex(anomalies, "progression")]
+ ))
+ if "regression" in anomalies:
+ l_reg.append((
+ _create_test_name(test).replace("-ndrpdr", "-pdr"),
+ x_axis[_get_rindex(anomalies, "regression")]
+ ))
+ else: # mrr
+ tst_data = tst_data.dropna(
+ subset=["result_receive_rate_rate_avg", ]
+ )
+ if tst_data.empty:
+ continue
+ try:
+ anomalies, _, _ = classify_anomalies({
+ k: v for k, v in zip(
+ x_axis,
+ tst_data["result_receive_rate_rate_avg"].\
+ tolist()
+ )
+ })
+ except ValueError:
+ continue
+ if "progression" in anomalies:
+ l_prog.append((
+ _create_test_name(test),
+ x_axis[_get_rindex(anomalies, "progression")]
+ ))
+ if "regression" in anomalies:
+ l_reg.append((
+ _create_test_name(test),
+ x_axis[_get_rindex(anomalies, "regression")]
+ ))
+
+ tst_info["regressions"].append(
+ sorted(l_reg, key=lambda k: k[1], reverse=True))
+ tst_info["progressions"].append(
+ sorted(l_prog, key=lambda k: k[1], reverse=True))
+
+ self._data = pd.DataFrame.from_dict(tst_info)
+
+ # Read from files:
+ self._html_layout = str()
+ self._tooltips = dict()
+
+ try:
+ with open(self._html_layout_file, "r") as file_read:
+ self._html_layout = file_read.read()
+ except IOError as err:
+ raise RuntimeError(
+ f"Not possible to open the file {self._html_layout_file}\n{err}"
+ )
+
+ try:
+ with open(self._tooltip_file, "r") as file_read:
+ self._tooltips = load(file_read, Loader=FullLoader)
+ except IOError as err:
+ logging.warning(
+ f"Not possible to open the file {self._tooltip_file}\n{err}"
+ )
+ except YAMLError as err:
+ logging.warning(
+ f"An error occurred while parsing the specification file "
+ f"{self._tooltip_file}\n{err}"
+ )
+
+ self._default_period = C.NEWS_SHORT
+ self._default_active = (False, True, False)
+ self._default_table = \
+ table_summary(self._data, self._jobs, self._default_period)
+
+ # Callbacks:
+ if self._app is not None and hasattr(self, 'callbacks'):
+ self.callbacks(self._app)
+
+ @property
+ def html_layout(self) -> dict:
+ return self._html_layout
+
+ def add_content(self):
+ """Top level method which generated the web page.
+
+ It generates:
+ - Store for user input data,
+ - Navigation bar,
+ - Main area with control panel and ploting area.
+
+ If no HTML layout is provided, an error message is displayed instead.
+
+ :returns: The HTML div with the whole page.
+ :rtype: html.Div
+ """
+
+ if self.html_layout:
+ return html.Div(
+ id="div-main",
+ className="small",
+ children=[
+ dcc.Location(id="url", refresh=False),
+ dbc.Row(
+ id="row-navbar",
+ class_name="g-0",
+ children=[
+ self._add_navbar(),
+ ]
+ ),
+ dbc.Row(
+ id="row-main",
+ class_name="g-0",
+ children=[
+ self._add_ctrl_col(),
+ self._add_plotting_col(),
+ ]
+ )
+ ]
+ )
+ else:
+ return html.Div(
+ id="div-main-error",
+ children=[
+ dbc.Alert(
+ [
+ "An Error Occured",
+ ],
+ color="danger",
+ ),
+ ]
+ )
+
+ def _add_navbar(self):
+ """Add nav element with navigation panel. It is placed on the top.
+
+ :returns: Navigation bar.
+ :rtype: dbc.NavbarSimple
+ """
+
+ return dbc.NavbarSimple(
+ id="navbarsimple-main",
+ children=[
+ dbc.NavItem(
+ dbc.NavLink(
+ "Continuous Performance News",
+ disabled=True,
+ external_link=True,
+ href="#"
+ )
+ )
+ ],
+ brand="Dashboard",
+ brand_href="/",
+ brand_external_link=True,
+ class_name="p-2",
+ fluid=True,
+ )
+
+ def _add_ctrl_col(self) -> dbc.Col:
+ """Add column with control panel. It is placed on the left side.
+
+ :returns: Column with the control panel.
+ :rtype: dbc.Col
+ """
+
+ return dbc.Col(
+ id="col-controls",
+ children=[
+ self._add_ctrl_panel(),
+ ],
+ )
+
+ def _add_plotting_col(self) -> dbc.Col:
+ """Add column with tables. It is placed on the right side.
+
+ :returns: Column with tables.
+ :rtype: dbc.Col
+ """
+
+ return dbc.Col(
+ id="col-plotting-area",
+ children=[
+ dcc.Loading(
+ children=[
+ dbc.Row( # Failed tests
+ id="row-table",
+ class_name="g-0 p-2",
+ children=self._default_table
+ ),
+ dbc.Row(
+ class_name="g-0 p-2",
+ align="center",
+ justify="start",
+ children=[
+ dbc.InputGroup(
+ class_name="me-1",
+ children=[
+ dbc.InputGroupText(
+ style=C.URL_STYLE,
+ children=show_tooltip(
+ self._tooltips,
+ "help-url", "URL",
+ "input-url"
+ )
+ ),
+ dbc.Input(
+ id="input-url",
+ readonly=True,
+ type="url",
+ style=C.URL_STYLE,
+ value=""
+ )
+ ]
+ )
+ ]
+ )
+ ]
+ )
+ ],
+ width=9,
+ )
+
+ def _add_ctrl_panel(self) -> dbc.Row:
+ """Add control panel.
+
+ :returns: Control panel.
+ :rtype: dbc.Row
+ """
+ return dbc.Row(
+ id="row-ctrl-panel",
+ class_name="g-0",
+ children=[
+ dbc.Row(
+ class_name="g-0 p-2",
+ children=[
+ dbc.Row(
+ class_name="g-0",
+ children=[
+ dbc.Label(
+ class_name="g-0",
+ children=show_tooltip(self._tooltips,
+ "help-summary-period", "Window")
+ ),
+ dbc.Row(
+ dbc.ButtonGroup(
+ id="bg-time-period",
+ class_name="g-0",
+ children=[
+ dbc.Button(
+ id="period-last",
+ children="Last Run",
+ className="me-1",
+ outline=True,
+ color="info"
+ ),
+ dbc.Button(
+ id="period-short",
+ children=\
+ f"Last {C.NEWS_SHORT} Runs",
+ className="me-1",
+ outline=True,
+ active=True,
+ color="info"
+ ),
+ dbc.Button(
+ id="period-long",
+ children="All Runs",
+ className="me-1",
+ outline=True,
+ color="info"
+ )
+ ]
+ )
+ )
+ ]
+ )
+ ]
+ )
+ ]
+ )
+
+ def callbacks(self, app):
+ """Callbacks for the whole application.
+
+ :param app: The application.
+ :type app: Flask
+ """
+
+ @app.callback(
+ Output("row-table", "children"),
+ Output("input-url", "value"),
+ Output("period-last", "active"),
+ Output("period-short", "active"),
+ Output("period-long", "active"),
+ Input("period-last", "n_clicks"),
+ Input("period-short", "n_clicks"),
+ Input("period-long", "n_clicks"),
+ Input("url", "href")
+ )
+ def _update_application(btn_last: int, btn_short: int, btn_long: int,
+ href: str) -> tuple:
+ """Update the application when the event is detected.
+
+ :returns: New values for web page elements.
+ :rtype: tuple
+ """
+
+ _, _, _ = btn_last, btn_short, btn_long
+
+ periods = {
+ "period-last": C.NEWS_LAST,
+ "period-short": C.NEWS_SHORT,
+ "period-long": C.NEWS_LONG
+ }
+ actives = {
+ "period-last": (True, False, False),
+ "period-short": (False, True, False),
+ "period-long": (False, False, True)
+ }
+
+ # Parse the url:
+ parsed_url = url_decode(href)
+ if parsed_url:
+ url_params = parsed_url["params"]
+ else:
+ url_params = None
+
+ trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
+ if trigger_id == "url" and url_params:
+ trigger_id = url_params.get("period", list())[0]
+
+ period = periods.get(trigger_id, self._default_period)
+ active = actives.get(trigger_id, self._default_active)
+
+ ret_val = [
+ table_summary(self._data, self._jobs, period),
+ gen_new_url(parsed_url, {"period": trigger_id})
+ ]
+ ret_val.extend(active)
+ return ret_val
Code Review / csit.git / blobdiff