UTI: Add Summary to News

Change-Id: Ifa3882b36976183f628d2e3171061945ab33bf29
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/resources/tools/dash/app/pal/__init__.py b/resources/tools/dash/app/pal/__init__.py
index 0eb2a4e..1ea6db0 100644 (file)
--- a/resources/tools/dash/app/pal/__init__.py
+++ b/resources/tools/dash/app/pal/__init__.py
@@ -27,9 +27,9 @@ def init_app():
     """
 
     logging.basicConfig(
     """
 
     logging.basicConfig(

-        format=u"%(asctime)s: %(levelname)s: %(message)s",
-        datefmt=u"%Y/%m/%d %H:%M:%S",
-        level=logging.INFO
+        format=C.LOG_FORMAT,
+        datefmt=C.LOG_DATE_FORMAT,
+        level=C.LOG_LEVEL

     )
 
     logging.info("Application started.")
     )
 
     logging.info("Application started.")

diff --git a/resources/tools/dash/app/pal/news/layout.py b/resources/tools/dash/app/pal/news/layout.py
index 73fabdf..6e59831 100644 (file)
--- a/resources/tools/dash/app/pal/news/layout.py
+++ b/resources/tools/dash/app/pal/news/layout.py
@@ -33,7 +33,7 @@ from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url, \
     set_job_params
 from ..utils.url_processing import url_decode
 from ..data.data import Data
     set_job_params
 from ..utils.url_processing import url_decode
 from ..data.data import Data

-from .tables import table_news
+from .tables import table_news, table_summary

 
 
 class Layout:
 
 
 class Layout:


@@ -77,7 +77,7 @@ class Layout:
         df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
 
         # Prepare information for the control panel:
         df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
 
         # Prepare information for the control panel:

-        jobs = sorted(list(df_tst_info["job"].unique()))
+        self._jobs = sorted(list(df_tst_info["job"].unique()))

         d_job_info = {
             "job": list(),
             "dut": list(),
         d_job_info = {
             "job": list(),
             "dut": list(),


@@ -85,7 +85,7 @@ class Layout:
             "cadence": list(),
             "tbed": list()
         }
             "cadence": list(),
             "tbed": list()
         }

-        for job in jobs:
+        for job in self._jobs:

             lst_job = job.split("-")
             d_job_info["job"].append(job)
             d_job_info["dut"].append(lst_job[1])
             lst_job = job.split("-")
             d_job_info["job"].append(job)
             d_job_info["dut"].append(lst_job[1])


@@ -118,7 +118,9 @@ class Layout:
             "regressions": list(),
             "progressions": list()
         }
             "regressions": list(),
             "progressions": list()
         }

-        for job in jobs:
+        logging.debug("Processing jobs ...")
+        for job in self._jobs:
+            logging.debug(f"+ {job}")

             # Create lists of failed tests:
             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
             last_build = max(df_job["build"].unique())
             # Create lists of failed tests:
             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
             last_build = max(df_job["build"].unique())


@@ -254,7 +256,8 @@ class Layout:
                 f"{self._tooltip_file}\n{err}"
             )
 
                 f"{self._tooltip_file}\n{err}"
             )
 

-        self._default_tab_failed = table_news(self.data, self._default["job"])
+        self._default_tab_failed = \
+            table_news(self.data, self._default["job"], C.NEWS_TIME_PERIOD)

 
         # Callbacks:
         if self._app is not None and hasattr(self, 'callbacks'):
 
         # Callbacks:
         if self._app is not None and hasattr(self, 'callbacks'):


@@ -487,6 +490,25 @@ class Layout:
                                 )
                             ]
                         ),
                                 )
                             ]
                         ),

+                        dbc.Row(
+                            class_name="gy-1 p-0",
+                            children=[
+                                dbc.ButtonGroup(
+                                    [
+                                        dbc.Button(
+                                            id="btn-summary",
+                                            children=(
+                                                f"Show Summary from the last "
+                                                f"{C.NEWS_SUMMARY_PERIOD} Days"
+                                            ),
+                                            class_name="me-1",
+                                            color="info"
+                                        )
+                                    ],
+                                    size="md",
+                                )
+                            ]
+                        ),

                         dbc.Row(
                             class_name="gy-1",
                             children=[
                         dbc.Row(
                             class_name="gy-1",
                             children=[


@@ -598,10 +620,11 @@ class Layout:
             Input("ri-ttypes", "value"),
             Input("ri-cadences", "value"),
             Input("dd-tbeds", "value"),
             Input("ri-ttypes", "value"),
             Input("ri-cadences", "value"),
             Input("dd-tbeds", "value"),

-            Input("url", "href")
+            Input("url", "href"),
+            Input("btn-summary", "n_clicks")

         )
         def _update_application(cp_data: dict, dut: str, ttype: str,
         )
         def _update_application(cp_data: dict, dut: str, ttype: str,

-                cadence:str, tbed: str, href: str) -> tuple:
+                cadence:str, tbed: str, href: str, btn_all: int) -> tuple:

             """Update the application when the event is detected.
 
             :param cp_data: Current status of the control panel stored in
             """Update the application when the event is detected.
 
             :param cp_data: Current status of the control panel stored in


@@ -630,6 +653,8 @@ class Layout:
             else:
                 url_params = None
 
             else:
                 url_params = None
 

+            show_summary = False
+

             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
             if trigger_id == "ri-duts":
                 ttype_opts = generate_options(get_ttypes(self.job_info, dut))
             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
             if trigger_id == "ri-duts":
                 ttype_opts = generate_options(get_ttypes(self.job_info, dut))


@@ -682,25 +707,37 @@ class Layout:
                 # TODO: Add verification
                 if url_params:
                     new_job = url_params.get("job", list())[0]
                 # TODO: Add verification
                 if url_params:
                     new_job = url_params.get("job", list())[0]

-                    if new_job:
+                    if new_job and new_job != "all":

                         job_params = set_job_params(self.job_info, new_job)
                         ctrl_panel = self.ControlPanel(None, job_params)
                         job_params = set_job_params(self.job_info, new_job)
                         ctrl_panel = self.ControlPanel(None, job_params)

+                    if new_job and new_job == "all":
+                        show_summary = True

                 else:
                     ctrl_panel = self.ControlPanel(cp_data, self.default)
                 else:
                     ctrl_panel = self.ControlPanel(cp_data, self.default)

+            elif trigger_id == "btn-summary":
+                show_summary = True

 
 

-            job = get_job(
-                self.job_info,
-                ctrl_panel.get("ri-duts-value"),
-                ctrl_panel.get("ri-ttypes-value"),
-                ctrl_panel.get("ri-cadences-value"),
-                ctrl_panel.get("dd-tbeds-value")
-            )
-            ctrl_panel.set({"al-job-children": job})
-            tab_failed = table_news(self.data, job)
+            if show_summary:
+                ctrl_panel.set({
+                    "al-job-children": \
+                        f"Summary from the last {C.NEWS_SUMMARY_PERIOD} days"
+                })
+                job = "all"
+                tables = table_summary(self.data, self._jobs)
+            else:
+                job = get_job(
+                    self.job_info,
+                    ctrl_panel.get("ri-duts-value"),
+                    ctrl_panel.get("ri-ttypes-value"),
+                    ctrl_panel.get("ri-cadences-value"),
+                    ctrl_panel.get("dd-tbeds-value")
+                )
+                ctrl_panel.set({"al-job-children": job})
+                tables = table_news(self.data, job, C.NEWS_TIME_PERIOD)

 
             ret_val = [
                 ctrl_panel.panel,
 
             ret_val = [
                 ctrl_panel.panel,

-                tab_failed,
+                tables,

                 gen_new_url(parsed_url, {"job": job})
             ]
             ret_val.extend(ctrl_panel.values())
                 gen_new_url(parsed_url, {"job": job})
             ]
             ret_val.extend(ctrl_panel.values())

diff --git a/resources/tools/dash/app/pal/news/tables.py b/resources/tools/dash/app/pal/news/tables.py
index 1a6c7d2..04b4fc9 100644 (file)
--- a/resources/tools/dash/app/pal/news/tables.py
+++ b/resources/tools/dash/app/pal/news/tables.py
@@ -17,10 +17,88 @@
 import pandas as pd
 import dash_bootstrap_components as dbc
 
 import pandas as pd
 import dash_bootstrap_components as dbc
 

+from datetime import datetime, timedelta
+from dash import html
+

 from ..utils.constants import Constants as C
 
 
 from ..utils.constants import Constants as C
 
 

-def table_news(data: pd.DataFrame, job: str) -> list:
+def _table_info(job_data: pd.DataFrame) -> dbc.Table:
+    """Generates table with info about the job.
+
+    :param job_data: Dataframe with information about the job.
+    :type job_data: pandas.DataFrame
+    :returns: Table with job info.
+    :rtype: dbc.Table
+    """
+    return dbc.Table.from_dataframe(
+        pd.DataFrame.from_dict(
+            {
+                "Job": job_data["job"],
+                "Last Build": job_data["build"],
+                "Date": job_data["start"],
+                "DUT": job_data["dut_type"],
+                "DUT Version": job_data["dut_version"],
+                "Hosts": ", ".join(job_data["hosts"].to_list()[0])
+            }
+        ),
+        bordered=True,
+        striped=True,
+        hover=True,
+        size="sm",
+        color="info"
+    )
+
+
+def _table_failed(job_data: pd.DataFrame, failed: list) -> dbc.Table:
+    """Generates table with failed tests from the last run of the job.
+
+    :param job_data: Dataframe with information about the job.
+    :param failed: List of failed tests.
+    :type job_data: pandas.DataFrame
+    :type failed: list
+    :returns: Table with fialed tests.
+    :rtype: dbc.Table
+    """
+    return dbc.Table.from_dataframe(
+        pd.DataFrame.from_dict(
+            {
+                (
+                    f"Last Failed Tests on "
+                    f"{job_data['start'].values[0]} ({len(failed)})"
+                ): failed
+            }
+        ),
+        bordered=True,
+        striped=True,
+        hover=True,
+        size="sm",
+        color="danger"
+    )
+
+
+def _table_gressions(itms: dict, color: str) -> dbc.Table:
+    """Generates table with regressions.
+
+    :param itms: Dictionary with items (regressions or progressions) and their
+        last occurence.
+    :param color: Color of the table.
+    :type regressions: dict
+    :type color: str
+    :returns: The table with regressions.
+    :rtype: dbc.Table
+    """
+    return dbc.Table.from_dataframe(
+        pd.DataFrame.from_dict(itms),
+        bordered=True,
+        striped=True,
+        hover=True,
+        size="sm",
+        color=color
+    )
+
+
+def table_news(data: pd.DataFrame, job: str, period: int) -> list:

     """Generates the tables with news:
     1. Falied tests from the last run
     2. Regressions and progressions calculated from the last C.NEWS_TIME_PERIOD
     """Generates the tables with news:
     1. Falied tests from the last run
     2. Regressions and progressions calculated from the last C.NEWS_TIME_PERIOD


@@ -29,57 +107,64 @@ def table_news(data: pd.DataFrame, job: str) -> list:
     :param data: Trending data with calculated annomalies to be displayed in the
         tables.
     :param job: The job name.
     :param data: Trending data with calculated annomalies to be displayed in the
         tables.
     :param job: The job name.

+    :param period: The time period (nr of days from now) taken into account.

     :type data: pandas.DataFrame
     :type job: str
     :type data: pandas.DataFrame
     :type job: str

+    :type period: int
+    :returns: List of tables.
+    :rtype: list

     """
 
     """
 

+    last_day = datetime.utcnow() - timedelta(days=period)
+    r_list = list()

     job_data = data.loc[(data["job"] == job)]
     job_data = data.loc[(data["job"] == job)]

+    r_list.append(_table_info(job_data))
+

     failed = job_data["failed"].to_list()[0]
     failed = job_data["failed"].to_list()[0]

-    regressions = {"Test Name": list(), "Last Regression": list()}
+    if failed:
+        r_list.append(_table_failed(job_data, failed))
+
+    title = f"Regressions in the last {period} days"
+    regressions = {title: list(), "Last Regression": list()}

     for itm in job_data["regressions"].to_list()[0]:
     for itm in job_data["regressions"].to_list()[0]:

-        regressions["Test Name"].append(itm[0])
-        regressions["Last Regression"].append(itm[1].strftime('%Y-%m-%d %H:%M'))
-    progressions = {"Test Name": list(), "Last Progression": list()}
+        if itm[1] < last_day:
+            break
+        regressions[title].append(itm[0])
+        regressions["Last Regression"].append(
+            itm[1].strftime('%Y-%m-%d %H:%M'))
+    if regressions["Last Regression"]:
+        r_list.append(_table_gressions(regressions, "warning"))
+
+    title = f"Progressions in the last {period} days"
+    progressions = {title: list(), "Last Progression": list()}

     for itm in job_data["progressions"].to_list()[0]:
     for itm in job_data["progressions"].to_list()[0]:

-        progressions["Test Name"].append(itm[0])
+        if itm[1] < last_day:
+            break
+        progressions[title].append(itm[0])

         progressions["Last Progression"].append(
             itm[1].strftime('%Y-%m-%d %H:%M'))
         progressions["Last Progression"].append(
             itm[1].strftime('%Y-%m-%d %H:%M'))

+    if progressions["Last Progression"]:
+        r_list.append(_table_gressions(progressions, "success"))

 
 

-    return [
-        dbc.Table.from_dataframe(pd.DataFrame.from_dict({
-            "Job": job_data["job"],
-            "Last Build": job_data["build"],
-            "Date": job_data["start"],
-            "DUT": job_data["dut_type"],
-            "DUT Version": job_data["dut_version"],
-            "Hosts": ", ".join(job_data["hosts"].to_list()[0])
-        }), bordered=True, striped=True, hover=True, size="sm", color="light"),
-        dbc.Table.from_dataframe(pd.DataFrame.from_dict({
-            (
-                f"Last Failed Tests on "
-                f"{job_data['start'].values[0]} ({len(failed)})"
-            ): failed
-        }), bordered=True, striped=True, hover=True, size="sm", color="light"),
-        dbc.Label(
-            class_name="p-0",
-            size="lg",
-            children=(
-                f"Regressions during the last {C.NEWS_TIME_PERIOD} days "
-                f"({len(regressions['Test Name'])})"
-            )
-        ),
-        dbc.Table.from_dataframe(
-            pd.DataFrame.from_dict(regressions),
-            bordered=True, striped=True, hover=True, size="sm", color="light"),
-        dbc.Label(
-            class_name="p-0",
-            size="lg",
-            children=(
-                f"Progressions during the last {C.NEWS_TIME_PERIOD} days "
-                f"({len(progressions['Test Name'])})"
-            )
-        ),
-        dbc.Table.from_dataframe(
-            pd.DataFrame.from_dict(progressions),
-            bordered=True, striped=True, hover=True, size="sm", color="light")
-    ]
+    return r_list
+
+
+def table_summary(data: pd.DataFrame, jobs: list) -> list:
+    """Generates summary (failed tests, regressions and progressions) from the
+    last week.
+
+    :param data: Trending data with calculated annomalies to be displayed in the
+        tables.
+    :param jobs: List of jobs.
+    :type data: pandas.DataFrame
+    :type job: str
+    :returns: List of tables.
+    :rtype: list
+    """
+
+    r_list = list()
+    for job in jobs:
+        r_list.extend(table_news(data, job, C.NEWS_SUMMARY_PERIOD))
+        r_list.append(html.Div(html.P(" ")))
+
+    return r_list

diff --git a/resources/tools/dash/app/pal/utils/constants.py b/resources/tools/dash/app/pal/utils/constants.py
index b95a8f5..1f31185 100644 (file)
--- a/resources/tools/dash/app/pal/utils/constants.py
+++ b/resources/tools/dash/app/pal/utils/constants.py
@@ -17,7 +17,7 @@
 does not need to be hard coded here, but can be read from environment variables.
 """
 
 does not need to be hard coded here, but can be read from environment variables.
 """
 

-
+import logging

 import dash_bootstrap_components as dbc
 
 from dash import html
 import dash_bootstrap_components as dbc
 
 from dash import html


@@ -30,6 +30,11 @@ class Constants:
     ############################################################################
     # General, application wide constants.
 
     ############################################################################
     # General, application wide constants.
 

+    # Logging settings.
+    LOG_LEVEL = logging.INFO
+    LOG_FORMAT = "%(asctime)s: %(levelname)s: %(message)s"
+    LOG_DATE_FORMAT = "%Y/%m/%d %H:%M:%S"
+

     # The application title.
     TITLE = "FD.io CSIT"
 
     # The application title.
     TITLE = "FD.io CSIT"
 


@@ -256,6 +261,9 @@ class Constants:
     # Time period for regressions and progressions.
     NEWS_TIME_PERIOD = TIME_PERIOD  # [days]
 
     # Time period for regressions and progressions.
     NEWS_TIME_PERIOD = TIME_PERIOD  # [days]
 

+    # Time period for summary tables.
+    NEWS_SUMMARY_PERIOD = 7  # [days]
+

     ############################################################################
     # Report.
 
     ############################################################################
     # Report.