feat(uti): Statistics - add url

[csit.git] / resources / tools / dash / app / pal / stats / layout.py

diff --git a/resources/tools/dash/app/pal/stats/layout.py b/resources/tools/dash/app/pal/stats/layout.py
index 0c5c22e..d265145 100644 (file)
--- a/resources/tools/dash/app/pal/stats/layout.py
+++ b/resources/tools/dash/app/pal/stats/layout.py
@@ -14,6 +14,8 @@
 """Plotly Dash HTML layout override.
 """
 
 """Plotly Dash HTML layout override.
 """
 

+import logging
+import urllib

 import pandas as pd
 import dash_bootstrap_components as dbc
 
 import pandas as pd
 import dash_bootstrap_components as dbc
 


@@ -21,21 +23,24 @@ from flask import Flask
 from dash import dcc
 from dash import html
 from dash import callback_context, no_update
 from dash import dcc
 from dash import html
 from dash import callback_context, no_update

-from dash import Input, Output
+from dash import Input, Output, State

 from dash.exceptions import PreventUpdate
 from yaml import load, FullLoader, YAMLError
 from datetime import datetime, timedelta
 from dash.exceptions import PreventUpdate
 from yaml import load, FullLoader, YAMLError
 from datetime import datetime, timedelta

+from copy import deepcopy

 
 from ..data.data import Data
 
 from ..data.data import Data

-from .graphs import graph_statistics
+from .graphs import graph_statistics, select_data

 
 
 class Layout:
     """
     """
 
 
 
 class Layout:
     """
     """
 

+    DEFAULT_JOB = "csit-vpp-perf-mrr-daily-master-2n-icx"
+

     def __init__(self, app: Flask, html_layout_file: str, spec_file: str,
     def __init__(self, app: Flask, html_layout_file: str, spec_file: str,

-        graph_layout_file: str, data_spec_file: str,
+        graph_layout_file: str, data_spec_file: str, tooltip_file: str,

         time_period: int=None) -> None:
         """
         """
         time_period: int=None) -> None:
         """
         """


@@ -46,6 +51,7 @@ class Layout:
         self._spec_file = spec_file
         self._graph_layout_file = graph_layout_file
         self._data_spec_file = data_spec_file
         self._spec_file = spec_file
         self._graph_layout_file = graph_layout_file
         self._data_spec_file = data_spec_file

+        self._tooltip_file = tooltip_file

         self._time_period = time_period
 
         # Read the data:
         self._time_period = time_period
 
         # Read the data:


@@ -67,7 +73,24 @@ class Layout:
         if self._time_period > data_time_period:
             self._time_period = data_time_period
 
         if self._time_period > data_time_period:
             self._time_period = data_time_period
 

-        self._jobs = sorted(list(data_stats["job"].unique()))
+        jobs = sorted(list(data_stats["job"].unique()))
+        job_info = {
+            "job": list(),
+            "dut": list(),
+            "ttype": list(),
+            "cadence": list(),
+            "tbed": list()
+        }
+        for job in jobs:
+            lst_job = job.split("-")
+            job_info["job"].append(job)
+            job_info["dut"].append(lst_job[1])
+            job_info["ttype"].append(lst_job[3])
+            job_info["cadence"].append(lst_job[4])
+            job_info["tbed"].append("-".join(lst_job[-2:]))
+        self.df_job_info = pd.DataFrame.from_dict(job_info)
+
+        self._default = self._set_job_params(self.DEFAULT_JOB)

 
         tst_info = {
             "job": list(),
 
         tst_info = {
             "job": list(),


@@ -78,7 +101,8 @@ class Layout:
             "passed": list(),
             "failed": list()
         }
             "passed": list(),
             "failed": list()
         }

-        for job in self._jobs:
+        for job in jobs:
+            # TODO: Add list of failed tests for each build

             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
             builds = df_job["build"].unique()
             for build in builds:
             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
             builds = df_job["build"].unique()
             for build in builds:


@@ -104,6 +128,7 @@ class Layout:
         # Read from files:
         self._html_layout = ""
         self._graph_layout = None
         # Read from files:
         self._html_layout = ""
         self._graph_layout = None

+        self._tooltips = dict()

 
         try:
             with open(self._html_layout_file, "r") as file_read:
 
         try:
             with open(self._html_layout_file, "r") as file_read:


@@ -124,12 +149,25 @@ class Layout:
         except YAMLError as err:
             raise RuntimeError(
                 f"An error occurred while parsing the specification file "
         except YAMLError as err:
             raise RuntimeError(
                 f"An error occurred while parsing the specification file "

-                f"{self._graph_layout_file}\n"
-                f"{err}"
+                f"{self._graph_layout_file}\n{err}"
+            )
+
+        try:
+            with open(self._tooltip_file, "r") as file_read:
+                self._tooltips = load(file_read, Loader=FullLoader)
+        except IOError as err:
+            logging.warning(
+                f"Not possible to open the file {self._tooltip_file}\n{err}"
+            )
+        except YAMLError as err:
+            logging.warning(
+                f"An error occurred while parsing the specification file "
+                f"{self._tooltip_file}\n{err}"

             )
 
             )
 

+

         self._default_fig_passed, self._default_fig_duration = graph_statistics(
         self._default_fig_passed, self._default_fig_duration = graph_statistics(

-            self.data, self.jobs[0], self.layout
+            self.data, self._default["job"], self.layout

         )
 
         # Callbacks:
         )
 
         # Callbacks:


@@ -149,12 +187,91 @@ class Layout:
         return self._graph_layout
 
     @property
         return self._graph_layout
 
     @property

-    def jobs(self) -> list:
-        return self._jobs
+    def time_period(self) -> int:
+        return self._time_period

 
     @property
 
     @property

-    def time_period(self):
-        return self._time_period
+    def default(self) -> any:
+        return self._default
+
+    def _get_duts(self) -> list:
+        """
+        """
+        return sorted(list(self.df_job_info["dut"].unique()))
+
+    def _get_ttypes(self, dut: str) -> list:
+        """
+        """
+        return sorted(list(self.df_job_info.loc[(
+            self.df_job_info["dut"] == dut
+        )]["ttype"].unique()))
+
+    def _get_cadences(self, dut: str, ttype: str) -> list:
+        """
+        """
+        return sorted(list(self.df_job_info.loc[(
+            (self.df_job_info["dut"] == dut) &
+            (self.df_job_info["ttype"] == ttype)
+        )]["cadence"].unique()))
+
+    def _get_test_beds(self, dut: str, ttype: str, cadence: str) -> list:
+        """
+        """
+        return sorted(list(self.df_job_info.loc[(
+            (self.df_job_info["dut"] == dut) &
+            (self.df_job_info["ttype"] == ttype) &
+            (self.df_job_info["cadence"] == cadence)
+        )]["tbed"].unique()))
+
+    def _get_job(self, dut, ttype, cadence, testbed):
+        """Get the name of a job defined by dut, ttype, cadence, testbed.
+
+        Input information comes from control panel.
+        """
+        return self.df_job_info.loc[(
+            (self.df_job_info["dut"] == dut) &
+            (self.df_job_info["ttype"] == ttype) &
+            (self.df_job_info["cadence"] == cadence) &
+            (self.df_job_info["tbed"] == testbed)
+        )]["job"].item()
+
+    def _set_job_params(self, job: str) -> dict:
+        """
+        """
+        lst_job = job.split("-")
+        return {
+            "job": job,
+            "dut": lst_job[1],
+            "ttype": lst_job[3],
+            "cadence": lst_job[4],
+            "tbed": "-".join(lst_job[-2:]),
+            "duts": self._generate_options(self._get_duts()),
+            "ttypes": self._generate_options(self._get_ttypes(lst_job[1])),
+            "cadences": self._generate_options(self._get_cadences(
+                lst_job[1], lst_job[3])),
+            "tbeds": self._generate_options(self._get_test_beds(
+                lst_job[1], lst_job[3], lst_job[4]))
+        }
+
+    def _show_tooltip(self, id: str, title: str) -> list:
+        """
+        """
+        return [
+            f"{title} ",
+            dbc.Badge(
+                id=id,
+                children="?",
+                pill=True,
+                color="white",
+                text_color="info",
+                class_name="border ms-1",
+            ),
+            dbc.Tooltip(
+                children=self._tooltips.get(id, str()),
+                target=id,
+                placement="auto"
+            )
+        ]

 
     def add_content(self):
         """
 
     def add_content(self):
         """


@@ -163,6 +280,8 @@ class Layout:
             return html.Div(
                 id="div-main",
                 children=[
             return html.Div(
                 id="div-main",
                 children=[

+                    dcc.Store(id="control-panel"),
+                    dcc.Location(id="url", refresh=False),

                     dbc.Row(
                         id="row-navbar",
                         class_name="g-0",
                     dbc.Row(
                         id="row-navbar",
                         class_name="g-0",


@@ -267,19 +386,37 @@ class Layout:
                         ])
                     ]
                 ),
                         ])
                     ]
                 ),

-                dbc.Row(  # Download
-                    id="row-btn-download",
+                dbc.Row(

                     class_name="g-0 p-2",
                     class_name="g-0 p-2",

+                    align="center",
+                    justify="start",

                     children=[
                     children=[

-                        dcc.Loading(children=[
-                            dbc.Button(
-                                id="btn-download-data",
-                                children=["Download Data"],
-                                class_name="me-1",
-                                color="info"
-                            ),
-                            dcc.Download(id="download-data")
-                        ])
+                        dbc.Col(  # Download
+                            width=2,
+                            children=[
+                                dcc.Loading(children=[
+                                    dbc.Button(
+                                        id="btn-download-data",
+                                        children=self._show_tooltip(
+                                            "help-download", "Download Data"),
+                                        class_name="me-1",
+                                        color="info"
+                                    ),
+                                    dcc.Download(id="download-data")
+                                ]),
+                            ]
+                        ),
+                        dbc.Col(  # Show URL
+                            width=10,
+                            children=[
+                                dbc.Card(
+                                    id="card-url",
+                                    body=True,
+                                    class_name="gy-2 p-0",
+                                    children=[]
+                                ),
+                            ]
+                        )

                     ]
                 )
             ],
                     ]
                 )
             ],


@@ -296,75 +433,329 @@ class Layout:
                 dbc.Row(
                     class_name="g-0 p-2",
                     children=[
                 dbc.Row(
                     class_name="g-0 p-2",
                     children=[

-                        dbc.Label("Choose the Trending Job"),
-                        dbc.RadioItems(
-                            id="ri_job",
-                            value=self.jobs[0],
-                            options=[
-                                {"label": i, "value": i} for i in self.jobs
+                        dbc.Row(
+                            class_name="gy-1",
+                            children=[
+                                dbc.Label(
+                                    class_name="p-0",
+                                    children=self._show_tooltip(
+                                        "help-dut", "Device under Test")
+                                ),
+                                dbc.Row(
+                                    dbc.RadioItems(
+                                        id="ri-duts",
+                                        inline=True,
+                                        value=self.default["dut"],
+                                        options=self.default["duts"]
+                                    )
+                                )
+                            ]
+                        ),
+                        dbc.Row(
+                            class_name="gy-1",
+                            children=[
+                                dbc.Label(
+                                    class_name="p-0",
+                                    children=self._show_tooltip(
+                                        "help-ttype", "Test Type"),
+                                ),
+                                dbc.RadioItems(
+                                    id="ri-ttypes",
+                                    inline=True,
+                                    value=self.default["ttype"],
+                                    options=self.default["ttypes"]
+                                )
+                            ]
+                        ),
+                        dbc.Row(
+                            class_name="gy-1",
+                            children=[
+                                dbc.Label(
+                                    class_name="p-0",
+                                    children=self._show_tooltip(
+                                        "help-cadence", "Cadence"),
+                                ),
+                                dbc.RadioItems(
+                                    id="ri-cadences",
+                                    inline=True,
+                                    value=self.default["cadence"],
+                                    options=self.default["cadences"]
+                                )
+                            ]
+                        ),
+                        dbc.Row(
+                            class_name="gy-1",
+                            children=[
+                                dbc.Label(
+                                    class_name="p-0",
+                                    children=self._show_tooltip(
+                                        "help-tbed", "Test Bed"),
+                                ),
+                                dbc.Select(
+                                    id="dd-tbeds",
+                                    placeholder="Select a test bed...",
+                                    value=self.default["tbed"],
+                                    options=self.default["tbeds"]
+                                )
+                            ]
+                        ),
+                        dbc.Row(
+                            class_name="gy-1",
+                            children=[
+                                dbc.Alert(
+                                    id="al-job",
+                                    color="info",
+                                    children=self.default["job"]
+                                )
+                            ]
+                        ),
+                        dbc.Row(
+                            class_name="g-0 p-2",
+                            children=[
+                                dbc.Label(
+                                    class_name="gy-1",
+                                    children=self._show_tooltip(
+                                        "help-time-period", "Time Period"),
+                                ),
+                                dcc.DatePickerRange(
+                                    id="dpr-period",
+                                    className="d-flex justify-content-center",
+                                    min_date_allowed=\
+                                        datetime.utcnow() - timedelta(
+                                            days=self.time_period),
+                                    max_date_allowed=datetime.utcnow(),
+                                    initial_visible_month=datetime.utcnow(),
+                                    start_date=\
+                                        datetime.utcnow() - timedelta(
+                                            days=self.time_period),
+                                    end_date=datetime.utcnow(),
+                                    display_format="D MMM YY"
+                                )

                             ]
                         )
                     ]
                 ),
                             ]
                         )
                     ]
                 ),

-                dbc.Row(
-                    class_name="g-0 p-2",
-                    children=[
-                        dbc.Label("Choose the Time Period"),
-                        dcc.DatePickerRange(
-                            id="dpr-period",
-                            className="d-flex justify-content-center",
-                            min_date_allowed=\
-                                datetime.utcnow() - timedelta(
-                                    days=self.time_period),
-                            max_date_allowed=datetime.utcnow(),
-                            initial_visible_month=datetime.utcnow(),
-                            start_date=\
-                                datetime.utcnow() - timedelta(
-                                    days=self.time_period),
-                            end_date=datetime.utcnow(),
-                            display_format="D MMMM YY"
-                        )
-                    ]
-                )

             ]
         )
 
             ]
         )
 

+    class ControlPanel:
+        def __init__(self, panel: dict, default: dict) -> None:
+            self._defaults = {
+                "ri-ttypes-options": default["ttypes"],
+                "ri-cadences-options": default["cadences"],
+                "dd-tbeds-options": default["tbeds"],
+                "ri-duts-value": default["dut"],
+                "ri-ttypes-value": default["ttype"],
+                "ri-cadences-value": default["cadence"],
+                "dd-tbeds-value": default["tbed"],
+                "al-job-children": default["job"]
+            }
+            self._panel = deepcopy(self._defaults)
+            if panel:
+                for key in self._defaults:
+                    self._panel[key] = panel[key]
+
+        def set(self, kwargs: dict) -> None:
+            for key, val in kwargs.items():
+                if key in self._panel:
+                    self._panel[key] = val
+                else:
+                    raise KeyError(f"The key {key} is not defined.")
+
+        @property
+        def defaults(self) -> dict:
+            return self._defaults
+
+        @property
+        def panel(self) -> dict:
+            return self._panel
+
+        def get(self, key: str) -> any:
+            return self._panel[key]
+
+        def values(self) -> list:
+            return list(self._panel.values())
+
+    @staticmethod
+    def _generate_options(opts: list) -> list:
+        """
+        """
+        return [{"label": i, "value": i} for i in opts]
+

     def callbacks(self, app):
 
         @app.callback(
     def callbacks(self, app):
 
         @app.callback(

+            Output("control-panel", "data"),  # Store

             Output("graph-passed", "figure"),
             Output("graph-duration", "figure"),
             Output("graph-passed", "figure"),
             Output("graph-duration", "figure"),

-            Input("ri_job", "value"),
+            Output("card-url", "children"),
+            Output("ri-ttypes", "options"),
+            Output("ri-cadences", "options"),
+            Output("dd-tbeds", "options"),
+            Output("ri-duts", "value"),
+            Output("ri-ttypes", "value"),
+            Output("ri-cadences", "value"),
+            Output("dd-tbeds", "value"),
+            Output("al-job", "children"),
+            State("control-panel", "data"),  # Store
+            Input("ri-duts", "value"),
+            Input("ri-ttypes", "value"),
+            Input("ri-cadences", "value"),
+            Input("dd-tbeds", "value"),

             Input("dpr-period", "start_date"),
             Input("dpr-period", "end_date"),
             Input("dpr-period", "start_date"),
             Input("dpr-period", "end_date"),

-            prevent_initial_call=True
+            Input("url", "href")
+            # prevent_initial_call=True

         )
         )

-        def _update_ctrl_panel(job:str, d_start: str, d_end: str) -> tuple:
+        def _update_ctrl_panel(cp_data: dict, dut:str, ttype: str, cadence:str,
+                tbed: str, start: str, end: str, href: str) -> tuple:

             """
             """
 
             """
             """
 

-            d_start = datetime(int(d_start[0:4]), int(d_start[5:7]),
-                int(d_start[8:10]))
-            d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10]))
+            ctrl_panel = self.ControlPanel(cp_data, self.default)

 
 

-            fig_passed, fig_duration = graph_statistics(
-                self.data, job, self.layout, d_start, d_end
+            start = datetime(int(start[0:4]), int(start[5:7]), int(start[8:10]))
+            end = datetime(int(end[0:4]), int(end[5:7]), int(end[8:10]))
+
+            parsed_url = urllib.parse.urlparse(href)
+            url = f"{parsed_url.netloc}{parsed_url.path}"
+            url_params = urllib.parse.parse_qs(parsed_url.fragment)
+
+            trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
+            if trigger_id == "ri-duts":
+                ttype_opts = self._generate_options(self._get_ttypes(dut))
+                ttype_val = ttype_opts[0]["value"]
+                cad_opts = self._generate_options(
+                    self._get_cadences(dut, ttype_val))
+                cad_val = cad_opts[0]["value"]
+                tbed_opts = self._generate_options(
+                    self._get_test_beds(dut, ttype_val, cad_val))
+                tbed_val = tbed_opts[0]["value"]
+                ctrl_panel.set({
+                    "ri-duts-value": dut,
+                    "ri-ttypes-options": ttype_opts,
+                    "ri-ttypes-value": ttype_val,
+                    "ri-cadences-options": cad_opts,
+                    "ri-cadences-value": cad_val,
+                    "dd-tbeds-options": tbed_opts,
+                    "dd-tbeds-value": tbed_val
+                })
+            elif trigger_id == "ri-ttypes":
+                cad_opts = self._generate_options(
+                    self._get_cadences(ctrl_panel.get("ri-duts-value"), ttype))
+                cad_val = cad_opts[0]["value"]
+                tbed_opts = self._generate_options(
+                    self._get_test_beds(ctrl_panel.get("ri-duts-value"),
+                    ttype, cad_val))
+                tbed_val = tbed_opts[0]["value"]
+                ctrl_panel.set({
+                    "ri-ttypes-value": ttype,
+                    "ri-cadences-options": cad_opts,
+                    "ri-cadences-value": cad_val,
+                    "dd-tbeds-options": tbed_opts,
+                    "dd-tbeds-value": tbed_val
+                })
+            elif trigger_id == "ri-cadences":
+                tbed_opts = self._generate_options(
+                    self._get_test_beds(ctrl_panel.get("ri-duts-value"),
+                    ctrl_panel.get("ri-ttypes-value"), cadence))
+                tbed_val = tbed_opts[0]["value"]
+                ctrl_panel.set({
+                    "ri-cadences-value": cadence,
+                    "dd-tbeds-options": tbed_opts,
+                    "dd-tbeds-value": tbed_val
+                })
+            elif trigger_id == "dd-tbeds":
+                ctrl_panel.set({
+                    "dd-tbeds-value": tbed
+                })
+            elif trigger_id == "dpr-period":
+                pass
+            elif trigger_id == "url":
+                # TODO: Add verification
+                if url_params:
+                    new_job = url_params.get("job", list())[0]
+                    new_start = url_params.get("start", list())[0]
+                    new_end = url_params.get("end", list())[0]
+                    if new_job and new_start and new_end:
+                        start = datetime(
+                            int(new_start[0:4]), int(new_start[5:7]),
+                            int(new_start[8:10]))
+                        end = datetime(
+                            int(new_end[0:4]), int(new_end[5:7]),
+                            int(new_end[8:10]))
+                        job_params = self._set_job_params(new_job)
+                        ctrl_panel = self.ControlPanel(None, job_params)
+                else:
+                    ctrl_panel = self.ControlPanel(cp_data, self.default)
+                    job = self._get_job(
+                        ctrl_panel.get("ri-duts-value"),
+                        ctrl_panel.get("ri-ttypes-value"),
+                        ctrl_panel.get("ri-cadences-value"),
+                        ctrl_panel.get("dd-tbeds-value")
+                    )
+
+            job = self._get_job(
+                ctrl_panel.get("ri-duts-value"),
+                ctrl_panel.get("ri-ttypes-value"),
+                ctrl_panel.get("ri-cadences-value"),
+                ctrl_panel.get("dd-tbeds-value")

             )
             )

+            url_params = {
+                "job": job,
+                "start": start,
+                "end": end
+            }

 
 

-            return fig_passed, fig_duration
+            ctrl_panel.set({"al-job-children": job})
+            fig_passed, fig_duration = graph_statistics(
+                self.data, job, self.layout, start, end)
+
+            ret_val = [
+                ctrl_panel.panel,
+                fig_passed,
+                fig_duration,
+                [
+                    dcc.Clipboard(
+                        target_id="card-url",
+                        title="Copy URL",
+                        style={"display": "inline-block"}
+                    ),
+                    f"{url}#{urllib.parse.urlencode(url_params)}"
+                ]
+            ]
+            ret_val.extend(ctrl_panel.values())
+            return ret_val

 
         @app.callback(
             Output("download-data", "data"),
 
         @app.callback(
             Output("download-data", "data"),

+            State("control-panel", "data"),  # Store
+            State("dpr-period", "start_date"),
+            State("dpr-period", "end_date"),

             Input("btn-download-data", "n_clicks"),
             prevent_initial_call=True
         )
             Input("btn-download-data", "n_clicks"),
             prevent_initial_call=True
         )

-        def _download_data(n_clicks):
+        def _download_data(cp_data: dict, start: str, end: str, n_clicks: int):

             """
             """
             """
             """

-            if not n_clicks:
+            if not (n_clicks):

                 raise PreventUpdate
 
                 raise PreventUpdate
 

-            return dcc.send_data_frame(self.data.to_csv, "statistics.csv")
+            ctrl_panel = self.ControlPanel(cp_data, self.default)
+
+            job = self._get_job(
+                ctrl_panel.get("ri-duts-value"),
+                ctrl_panel.get("ri-ttypes-value"),
+                ctrl_panel.get("ri-cadences-value"),
+                ctrl_panel.get("dd-tbeds-value")
+            )
+
+            start = datetime(int(start[0:4]), int(start[5:7]), int(start[8:10]))
+            end = datetime(int(end[0:4]), int(end[5:7]), int(end[8:10]))
+            data = select_data(self.data, job, start, end)
+            data = data.drop(columns=["job", ])
+
+            return dcc.send_data_frame(data.T.to_csv, f"{job}-stats.csv")

 
         @app.callback(
             Output("row-metadata", "children"),
 
         @app.callback(
             Output("row-metadata", "children"),