X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fdash%2Fapp%2Fpal%2Fstats%2Flayout.py;h=8df6d53bac7db7d39adda08b357da2c1909b8e26;hp=0ae83cf3a494e858b0822a4ffadd59dd48959b1f;hb=24e0d369e3b5637926269c01ebbfc78a185e85cc;hpb=cb61767fb2133890a786fc8f580109e9f8f3c5c5 diff --git a/resources/tools/dash/app/pal/stats/layout.py b/resources/tools/dash/app/pal/stats/layout.py index 0ae83cf3a4..8df6d53bac 100644 --- a/resources/tools/dash/app/pal/stats/layout.py +++ b/resources/tools/dash/app/pal/stats/layout.py @@ -14,42 +14,74 @@ """Plotly Dash HTML layout override. """ +import logging import pandas as pd import dash_bootstrap_components as dbc +from flask import Flask from dash import dcc from dash import html from dash import callback_context, no_update -from dash import Input, Output +from dash import Input, Output, State from dash.exceptions import PreventUpdate from yaml import load, FullLoader, YAMLError from datetime import datetime, timedelta +from copy import deepcopy +from ..utils.constants import Constants as C +from ..utils.utils import show_tooltip, gen_new_url, get_date, get_ttypes, \ + get_cadences, get_test_beds, get_job, generate_options, set_job_params +from ..utils.url_processing import url_decode from ..data.data import Data -from .graphs import graph_statistics +from .graphs import graph_statistics, select_data class Layout: - """ + """The layout of the dash app and the callbacks. """ - def __init__(self, app, html_layout_file, spec_file, graph_layout_file, - data_spec_file): - """ + def __init__(self, app: Flask, html_layout_file: str, + graph_layout_file: str, data_spec_file: str, tooltip_file: str, + time_period: int=None) -> None: + """Initialization: + - save the input parameters, + - read and pre-process the data, + - prepare data for the control panel, + - read HTML layout file, + - read tooltips from the tooltip file. + + :param app: Flask application running the dash application. + :param html_layout_file: Path and name of the file specifying the HTML + layout of the dash application. + :param graph_layout_file: Path and name of the file with layout of + plot.ly graphs. + :param data_spec_file: Path and name of the file specifying the data to + be read from parquets for this application. + :param tooltip_file: Path and name of the yaml file specifying the + tooltips. + :param time_period: It defines the time period for data read from the + parquets in days from now back to the past. + :type app: Flask + :type html_layout_file: str + :type graph_layout_file: str + :type data_spec_file: str + :type tooltip_file: str + :type time_period: int """ # Inputs self._app = app self._html_layout_file = html_layout_file - self._spec_file = spec_file self._graph_layout_file = graph_layout_file self._data_spec_file = data_spec_file + self._tooltip_file = tooltip_file + self._time_period = time_period # Read the data: data_stats, data_mrr, data_ndrpdr = Data( data_spec_file=self._data_spec_file, debug=True - ).read_stats() + ).read_stats(days=self._time_period) df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True) @@ -59,7 +91,29 @@ class Layout: data_stats = data_stats[~data_stats.job.str.contains("-iterative-")] data_stats = data_stats[["job", "build", "start_time", "duration"]] - self._jobs = sorted(list(data_stats["job"].unique())) + data_time_period = \ + (datetime.utcnow() - data_stats["start_time"].min()).days + if self._time_period > data_time_period: + self._time_period = data_time_period + + jobs = sorted(list(data_stats["job"].unique())) + d_job_info = { + "job": list(), + "dut": list(), + "ttype": list(), + "cadence": list(), + "tbed": list() + } + for job in jobs: + lst_job = job.split("-") + d_job_info["job"].append(job) + d_job_info["dut"].append(lst_job[1]) + d_job_info["ttype"].append(lst_job[3]) + d_job_info["cadence"].append(lst_job[4]) + d_job_info["tbed"].append("-".join(lst_job[-2:])) + self.job_info = pd.DataFrame.from_dict(d_job_info) + + self._default = set_job_params(self.job_info, C.STATS_DEFAULT_JOB) tst_info = { "job": list(), @@ -68,9 +122,10 @@ class Layout: "dut_version": list(), "hosts": list(), "passed": list(), - "failed": list() + "failed": list(), + "lst_failed": list() } - for job in self._jobs: + for job in jobs: df_job = df_tst_info.loc[(df_tst_info["job"] == job)] builds = df_job["build"].unique() for build in builds: @@ -81,21 +136,32 @@ class Layout: tst_info["dut_version"].append(df_build["dut_version"].iloc[-1]) tst_info["hosts"].append(df_build["hosts"].iloc[-1]) try: - passed = df_build.value_counts(subset='passed')[True] + passed = df_build.value_counts(subset="passed")[True] except KeyError: passed = 0 try: - failed = df_build.value_counts(subset='passed')[False] + failed = df_build.value_counts(subset="passed")[False] + failed_tests = df_build.loc[(df_build["passed"] == False)]\ + ["test_id"].to_list() + l_failed = list() + for tst in failed_tests: + lst_tst = tst.split(".") + suite = lst_tst[-2].replace("2n1l-", "").\ + replace("1n1l-", "").replace("2n-", "") + l_failed.append(f"{suite.split('-')[0]}-{lst_tst[-1]}") except KeyError: failed = 0 + l_failed = list() tst_info["passed"].append(passed) tst_info["failed"].append(failed) + tst_info["lst_failed"].append(sorted(l_failed)) self._data = data_stats.merge(pd.DataFrame.from_dict(tst_info)) # Read from files: self._html_layout = "" self._graph_layout = None + self._tooltips = dict() try: with open(self._html_layout_file, "r") as file_read: @@ -116,12 +182,25 @@ class Layout: except YAMLError as err: raise RuntimeError( f"An error occurred while parsing the specification file " - f"{self._graph_layout_file}\n" - f"{err}" + f"{self._graph_layout_file}\n{err}" ) + try: + with open(self._tooltip_file, "r") as file_read: + self._tooltips = load(file_read, Loader=FullLoader) + except IOError as err: + logging.warning( + f"Not possible to open the file {self._tooltip_file}\n{err}" + ) + except YAMLError as err: + logging.warning( + f"An error occurred while parsing the specification file " + f"{self._tooltip_file}\n{err}" + ) + + self._default_fig_passed, self._default_fig_duration = graph_statistics( - self.data, self.jobs[0], self.layout + self.data, self._default["job"], self.layout ) # Callbacks: @@ -141,16 +220,33 @@ class Layout: return self._graph_layout @property - def jobs(self) -> list: - return self._jobs + def time_period(self) -> int: + return self._time_period + + @property + def default(self) -> any: + return self._default def add_content(self): + """Top level method which generated the web page. + + It generates: + - Store for user input data, + - Navigation bar, + - Main area with control panel and ploting area. + + If no HTML layout is provided, an error message is displayed instead. + + :returns: The HTML div with the whole page. + :rtype: html.Div """ - """ + if self.html_layout: return html.Div( id="div-main", children=[ + dcc.Store(id="control-panel"), + dcc.Location(id="url", refresh=False), dbc.Row( id="row-navbar", class_name="g-0", @@ -195,6 +291,9 @@ class Layout: def _add_navbar(self): """Add nav element with navigation panel. It is placed on the top. + + :returns: Navigation bar. + :rtype: dbc.NavbarSimple """ return dbc.NavbarSimple( id="navbarsimple-main", @@ -217,6 +316,9 @@ class Layout: def _add_ctrl_col(self) -> dbc.Col: """Add column with controls. It is placed on the left side. + + :returns: Column with the control panel. + :rtype: dbc.Col """ return dbc.Col( id="col-controls", @@ -227,6 +329,9 @@ class Layout: def _add_plotting_col(self) -> dbc.Col: """Add column with plots and tables. It is placed on the right side. + + :returns: Column with tables. + :rtype: dbc.Col """ return dbc.Col( id="col-plotting-area", @@ -255,19 +360,51 @@ class Layout: ]) ] ), - dbc.Row( # Download - id="row-btn-download", + dbc.Row( class_name="g-0 p-2", + align="center", + justify="start", children=[ - dcc.Loading(children=[ - dbc.Button( - id="btn-download-data", - children=["Download Data"], - class_name="me-1", - color="info" - ), - dcc.Download(id="download-data") - ]) + dbc.Col( # Download + width=2, + children=[ + dcc.Loading(children=[ + dbc.Button( + id="btn-download-data", + children=show_tooltip(self._tooltips, + "help-download", "Download Data"), + class_name="me-1", + color="info" + ), + dcc.Download(id="download-data") + ]) + ] + ), + dbc.Col( # Show URL + width=10, + children=[ + dbc.InputGroup( + class_name="me-1", + children=[ + dbc.InputGroupText( + style=C.URL_STYLE, + children=show_tooltip( + self._tooltips, + "help-url", "URL", + "input-url" + ) + ), + dbc.Input( + id="input-url", + readonly=True, + type="url", + style=C.URL_STYLE, + value="" + ) + ] + ) + ] + ) ] ) ], @@ -275,7 +412,10 @@ class Layout: ) def _add_ctrl_panel(self) -> dbc.Row: - """ + """Add control panel. + + :returns: Control panel. + :rtype: dbc.Row """ return dbc.Row( id="row-ctrl-panel", @@ -284,72 +424,383 @@ class Layout: dbc.Row( class_name="g-0 p-2", children=[ - dbc.Label("Choose the Trending Job"), - dbc.RadioItems( - id="ri_job", - value=self.jobs[0], - options=[ - {"label": i, "value": i} for i in self.jobs + dbc.Row( + class_name="gy-1", + children=[ + dbc.Label( + class_name="p-0", + children=show_tooltip(self._tooltips, + "help-dut", "Device under Test") + ), + dbc.Row( + dbc.RadioItems( + id="ri-duts", + inline=True, + value=self.default["dut"], + options=self.default["duts"] + ) + ) + ] + ), + dbc.Row( + class_name="gy-1", + children=[ + dbc.Label( + class_name="p-0", + children=show_tooltip(self._tooltips, + "help-ttype", "Test Type"), + ), + dbc.RadioItems( + id="ri-ttypes", + inline=True, + value=self.default["ttype"], + options=self.default["ttypes"] + ) + ] + ), + dbc.Row( + class_name="gy-1", + children=[ + dbc.Label( + class_name="p-0", + children=show_tooltip(self._tooltips, + "help-cadence", "Cadence"), + ), + dbc.RadioItems( + id="ri-cadences", + inline=True, + value=self.default["cadence"], + options=self.default["cadences"] + ) + ] + ), + dbc.Row( + class_name="gy-1", + children=[ + dbc.Label( + class_name="p-0", + children=show_tooltip(self._tooltips, + "help-tbed", "Test Bed"), + ), + dbc.Select( + id="dd-tbeds", + placeholder="Select a test bed...", + value=self.default["tbed"], + options=self.default["tbeds"] + ) + ] + ), + dbc.Row( + class_name="gy-1", + children=[ + dbc.Alert( + id="al-job", + color="info", + children=self.default["job"] + ) + ] + ), + dbc.Row( + class_name="g-0 p-2", + children=[ + dbc.Label( + class_name="gy-1", + children=show_tooltip(self._tooltips, + "help-time-period", "Time Period"), + ), + dcc.DatePickerRange( + id="dpr-period", + className="d-flex justify-content-center", + min_date_allowed=\ + datetime.utcnow() - timedelta( + days=self.time_period), + max_date_allowed=datetime.utcnow(), + initial_visible_month=datetime.utcnow(), + start_date=\ + datetime.utcnow() - timedelta( + days=self.time_period), + end_date=datetime.utcnow(), + display_format="D MMM YY" + ) ] ) ] ), - dbc.Row( - class_name="g-0 p-2", - children=[ - dbc.Label("Choose the Time Period"), - dcc.DatePickerRange( - id="dpr-period", - className="d-flex justify-content-center", - min_date_allowed=\ - datetime.utcnow()-timedelta(days=180), - max_date_allowed=datetime.utcnow(), - initial_visible_month=datetime.utcnow(), - start_date=datetime.utcnow() - timedelta(days=180), - end_date=datetime.utcnow(), - display_format="D MMMM YY" - ) - ] - ) ] ) + class ControlPanel: + """A class representing the control panel. + """ + + def __init__(self, panel: dict, default: dict) -> None: + """Initialisation of the control pannel by default values. If + particular values are provided (parameter "panel") they are set + afterwards. + + :param panel: Custom values to be set to the control panel. + :param default: Default values to be set to the control panel. + :type panel: dict + :type defaults: dict + """ + + self._defaults = { + "ri-ttypes-options": default["ttypes"], + "ri-cadences-options": default["cadences"], + "dd-tbeds-options": default["tbeds"], + "ri-duts-value": default["dut"], + "ri-ttypes-value": default["ttype"], + "ri-cadences-value": default["cadence"], + "dd-tbeds-value": default["tbed"], + "al-job-children": default["job"] + } + self._panel = deepcopy(self._defaults) + if panel: + for key in self._defaults: + self._panel[key] = panel[key] + + def set(self, kwargs: dict) -> None: + """Set the values of the Control panel. + + :param kwargs: key - value pairs to be set. + :type kwargs: dict + :raises KeyError: If the key in kwargs is not present in the Control + panel. + """ + for key, val in kwargs.items(): + if key in self._panel: + self._panel[key] = val + else: + raise KeyError(f"The key {key} is not defined.") + + @property + def defaults(self) -> dict: + return self._defaults + + @property + def panel(self) -> dict: + return self._panel + + def get(self, key: str) -> any: + """Returns the value of a key from the Control panel. + + :param key: The key which value should be returned. + :type key: str + :returns: The value of the key. + :rtype: any + :raises KeyError: If the key in kwargs is not present in the Control + panel. + """ + return self._panel[key] + + def values(self) -> list: + """Returns the values from the Control panel as a list. + + :returns: The values from the Control panel. + :rtype: list + """ + return list(self._panel.values()) + + def callbacks(self, app): + """Callbacks for the whole application. + + :param app: The application. + :type app: Flask + """ @app.callback( + Output("control-panel", "data"), # Store Output("graph-passed", "figure"), Output("graph-duration", "figure"), - Input("ri_job", "value"), + Output("input-url", "value"), + Output("ri-ttypes", "options"), + Output("ri-cadences", "options"), + Output("dd-tbeds", "options"), + Output("ri-duts", "value"), + Output("ri-ttypes", "value"), + Output("ri-cadences", "value"), + Output("dd-tbeds", "value"), + Output("al-job", "children"), + State("control-panel", "data"), # Store + Input("ri-duts", "value"), + Input("ri-ttypes", "value"), + Input("ri-cadences", "value"), + Input("dd-tbeds", "value"), Input("dpr-period", "start_date"), Input("dpr-period", "end_date"), - prevent_initial_call=True + Input("url", "href") ) - def _update_ctrl_panel(job:str, d_start: str, d_end: str) -> tuple: - """ + def _update_ctrl_panel(cp_data: dict, dut: str, ttype: str, cadence:str, + tbed: str, start: str, end: str, href: str) -> tuple: + """Update the application when the event is detected. + + :param cp_data: Current status of the control panel stored in + browser. + :param dut: Input - DUT name. + :param ttype: Input - Test type. + :param cadence: Input - The cadence of the job. + :param tbed: Input - The test bed. + :param start: Date and time where the data processing starts. + :param end: Date and time where the data processing ends. + :param href: Input - The URL provided by the browser. + :type cp_data: dict + :type dut: str + :type ttype: str + :type cadence: str + :type tbed: str + :type start: str + :type end: str + :type href: str + :returns: New values for web page elements. + :rtype: tuple """ - d_start = datetime(int(d_start[0:4]), int(d_start[5:7]), - int(d_start[8:10])) - d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10])) + ctrl_panel = self.ControlPanel(cp_data, self.default) + + start = get_date(start) + end = get_date(end) - fig_passed, fig_duration = graph_statistics( - self.data, job, self.layout, d_start, d_end + # Parse the url: + parsed_url = url_decode(href) + if parsed_url: + url_params = parsed_url["params"] + else: + url_params = None + + trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0] + if trigger_id == "ri-duts": + ttype_opts = generate_options(get_ttypes(self.job_info, dut)) + ttype_val = ttype_opts[0]["value"] + cad_opts = generate_options(get_cadences( + self.job_info, dut, ttype_val)) + cad_val = cad_opts[0]["value"] + tbed_opts = generate_options(get_test_beds( + self.job_info, dut, ttype_val, cad_val)) + tbed_val = tbed_opts[0]["value"] + ctrl_panel.set({ + "ri-duts-value": dut, + "ri-ttypes-options": ttype_opts, + "ri-ttypes-value": ttype_val, + "ri-cadences-options": cad_opts, + "ri-cadences-value": cad_val, + "dd-tbeds-options": tbed_opts, + "dd-tbeds-value": tbed_val + }) + elif trigger_id == "ri-ttypes": + cad_opts = generate_options(get_cadences( + self.job_info, ctrl_panel.get("ri-duts-value"), ttype)) + cad_val = cad_opts[0]["value"] + tbed_opts = generate_options(get_test_beds( + self.job_info, ctrl_panel.get("ri-duts-value"), ttype, + cad_val)) + tbed_val = tbed_opts[0]["value"] + ctrl_panel.set({ + "ri-ttypes-value": ttype, + "ri-cadences-options": cad_opts, + "ri-cadences-value": cad_val, + "dd-tbeds-options": tbed_opts, + "dd-tbeds-value": tbed_val + }) + elif trigger_id == "ri-cadences": + tbed_opts = generate_options(get_test_beds( + self.job_info, ctrl_panel.get("ri-duts-value"), + ctrl_panel.get("ri-ttypes-value"), cadence)) + tbed_val = tbed_opts[0]["value"] + ctrl_panel.set({ + "ri-cadences-value": cadence, + "dd-tbeds-options": tbed_opts, + "dd-tbeds-value": tbed_val + }) + elif trigger_id == "dd-tbeds": + ctrl_panel.set({ + "dd-tbeds-value": tbed + }) + elif trigger_id == "dpr-period": + pass + elif trigger_id == "url": + # TODO: Add verification + if url_params: + new_job = url_params.get("job", list())[0] + new_start = url_params.get("start", list())[0] + new_end = url_params.get("end", list())[0] + if new_job and new_start and new_end: + start = get_date(new_start) + end = get_date(new_end) + job_params = set_job_params(self.job_info, new_job) + ctrl_panel = self.ControlPanel(None, job_params) + else: + ctrl_panel = self.ControlPanel(cp_data, self.default) + + job = get_job( + self.job_info, + ctrl_panel.get("ri-duts-value"), + ctrl_panel.get("ri-ttypes-value"), + ctrl_panel.get("ri-cadences-value"), + ctrl_panel.get("dd-tbeds-value") ) - return fig_passed, fig_duration + ctrl_panel.set({"al-job-children": job}) + fig_passed, fig_duration = graph_statistics(self.data, job, + self.layout, start, end) + + ret_val = [ + ctrl_panel.panel, + fig_passed, + fig_duration, + gen_new_url( + parsed_url, + { + "job": job, + "start": start, + "end": end + } + ) + ] + ret_val.extend(ctrl_panel.values()) + return ret_val @app.callback( Output("download-data", "data"), + State("control-panel", "data"), # Store + State("dpr-period", "start_date"), + State("dpr-period", "end_date"), Input("btn-download-data", "n_clicks"), prevent_initial_call=True ) - def _download_data(n_clicks): - """ + def _download_data(cp_data: dict, start: str, end: str, n_clicks: int): + """Download the data + + :param cp_data: Current status of the control panel stored in + browser. + :param start: Date and time where the data processing starts. + :param end: Date and time where the data processing ends. + :param n_clicks: Number of clicks on the button "Download". + :type cp_data: dict + :type start: str + :type end: str + :type n_clicks: int + :returns: dict of data frame content (base64 encoded) and meta data + used by the Download component. + :rtype: dict """ - if not n_clicks: + if not (n_clicks): raise PreventUpdate - return dcc.send_data_frame(self.data.to_csv, "statistics.csv") + ctrl_panel = self.ControlPanel(cp_data, self.default) + + job = get_job( + self.job_info, + ctrl_panel.get("ri-duts-value"), + ctrl_panel.get("ri-ttypes-value"), + ctrl_panel.get("ri-cadences-value"), + ctrl_panel.get("dd-tbeds-value") + ) + + data = select_data(self.data, job, get_date(start), get_date(end)) + data = data.drop(columns=["job", ]) + + return dcc.send_data_frame(data.T.to_csv, f"{job}-stats.csv") @app.callback( Output("row-metadata", "children"), @@ -360,7 +811,19 @@ class Layout: ) def _show_metadata_from_graphs( passed_data: dict, duration_data: dict) -> tuple: - """ + """Generates the data for the offcanvas displayed when a particular + point in a graph is clicked on. + + :param passed_data: The data from the clicked point in the graph + displaying the pass/fail data. + :param duration_data: The data from the clicked point in the graph + displaying the duration data. + :type passed_data: dict + :type duration data: dict + :returns: The data to be displayed on the offcanvas (job statistics + and the list of failed tests) and the information to show the + offcanvas. + :rtype: tuple(list, bool) """ if not (passed_data or duration_data): @@ -375,6 +838,26 @@ class Layout: elif trigger_id == "graph-duration": graph_data = duration_data["points"][0].get("text", "") if graph_data: + lst_graph_data = graph_data.split("
") + + # Prepare list of failed tests: + job = str() + build = str() + for itm in lst_graph_data: + if "csit-ref:" in itm: + job, build = itm.split(" ")[-1].split("/") + break + if job and build: + fail_tests = self.data.loc[ + (self.data["job"] == job) & + (self.data["build"] == build) + ]["lst_failed"].values[0] + if not fail_tests: + fail_tests = None + else: + fail_tests = None + + # Create the content of the offcanvas: metadata = [ dbc.Card( class_name="gy-2 p-0", @@ -399,7 +882,7 @@ class Layout: ), x.split(": ")[1] ] - ) for x in graph_data.split("
") + ) for x in lst_graph_data ], flush=True), ] @@ -407,6 +890,30 @@ class Layout: ] ) ] + + if fail_tests is not None: + metadata.append( + dbc.Card( + class_name="gy-2 p-0", + children=[ + dbc.CardHeader( + f"List of Failed Tests ({len(fail_tests)})" + ), + dbc.CardBody( + id="failed-tests", + class_name="p-0", + children=[dbc.ListGroup( + children=[ + dbc.ListGroupItem(x) \ + for x in fail_tests + ], + flush=True), + ] + ) + ] + ) + ) + open_canvas = True return metadata, open_canvas