C-Dash: Small changes in graphs processing

[csit.git] / csit.infra.dash / app / cdash / stats / layout.py

diff --git a/csit.infra.dash/app/cdash/stats/layout.py b/csit.infra.dash/app/cdash/stats/layout.py
index dace219..753eb37 100644 (file)
--- a/csit.infra.dash/app/cdash/stats/layout.py
+++ b/csit.infra.dash/app/cdash/stats/layout.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2022 Cisco and/or its affiliates.
+# Copyright (c) 2023 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -25,14 +25,12 @@ from dash import callback_context, no_update
 from dash import Input, Output, State
 from dash.exceptions import PreventUpdate
 from yaml import load, FullLoader, YAMLError
 from dash import Input, Output, State
 from dash.exceptions import PreventUpdate
 from yaml import load, FullLoader, YAMLError

-from datetime import datetime

 
 from ..utils.constants import Constants as C
 from ..utils.control_panel import ControlPanel
 from ..utils.utils import show_tooltip, gen_new_url, get_ttypes, get_cadences, \
     get_test_beds, get_job, generate_options, set_job_params
 from ..utils.url_processing import url_decode
 
 from ..utils.constants import Constants as C
 from ..utils.control_panel import ControlPanel
 from ..utils.utils import show_tooltip, gen_new_url, get_ttypes, get_cadences, \
     get_test_beds, get_job, generate_options, set_job_params
 from ..utils.url_processing import url_decode

-from ..data.data import Data

 from .graphs import graph_statistics, select_data
 
 
 from .graphs import graph_statistics, select_data
 
 


@@ -40,9 +38,15 @@ class Layout:
     """The layout of the dash app and the callbacks.
     """
 
     """The layout of the dash app and the callbacks.
     """
 

-    def __init__(self, app: Flask, html_layout_file: str,
-        graph_layout_file: str, data_spec_file: str, tooltip_file: str,
-        time_period: int=None) -> None:
+    def __init__(
+            self,
+            app: Flask,
+            data_stats: pd.DataFrame,
+            data_trending: pd.DataFrame,
+            html_layout_file: str,
+            graph_layout_file: str,
+            tooltip_file: str
+        ) -> None:

         """Initialization:
         - save the input parameters,
         - read and pre-process the data,
         """Initialization:
         - save the input parameters,
         - read and pre-process the data,


@@ -51,39 +55,27 @@ class Layout:
         - read tooltips from the tooltip file.
 
         :param app: Flask application running the dash application.
         - read tooltips from the tooltip file.
 
         :param app: Flask application running the dash application.

+        :param data_stats: Pandas dataframe with staistical data.
+        :param data_trending: Pandas dataframe with trending data.

         :param html_layout_file: Path and name of the file specifying the HTML
             layout of the dash application.
         :param graph_layout_file: Path and name of the file with layout of
             plot.ly graphs.
         :param html_layout_file: Path and name of the file specifying the HTML
             layout of the dash application.
         :param graph_layout_file: Path and name of the file with layout of
             plot.ly graphs.

-        :param data_spec_file: Path and name of the file specifying the data to
-            be read from parquets for this application.

         :param tooltip_file: Path and name of the yaml file specifying the
             tooltips.
         :param tooltip_file: Path and name of the yaml file specifying the
             tooltips.

-        :param time_period: It defines the time period for data read from the
-            parquets in days from now back to the past.

         :type app: Flask
         :type app: Flask

+        :type data_stats: pandas.DataFrame
+        :type data_trending: pandas.DataFrame

         :type html_layout_file: str
         :type graph_layout_file: str
         :type html_layout_file: str
         :type graph_layout_file: str

-        :type data_spec_file: str

         :type tooltip_file: str
         :type tooltip_file: str

-        :type time_period: int

         """
 
         # Inputs
         self._app = app
         self._html_layout_file = html_layout_file
         self._graph_layout_file = graph_layout_file
         """
 
         # Inputs
         self._app = app
         self._html_layout_file = html_layout_file
         self._graph_layout_file = graph_layout_file

-        self._data_spec_file = data_spec_file

         self._tooltip_file = tooltip_file
         self._tooltip_file = tooltip_file

-        self._time_period = time_period
-
-        # Read the data:
-        data_stats, data_mrr, data_ndrpdr = Data(
-            data_spec_file=self._data_spec_file,
-            debug=True
-        ).read_stats(days=self._time_period)
-
-        df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)

 
         # Pre-process the data:
         data_stats = data_stats[~data_stats.job.str.contains("-verify-")]
 
         # Pre-process the data:
         data_stats = data_stats[~data_stats.job.str.contains("-verify-")]


@@ -91,11 +83,6 @@ class Layout:
         data_stats = data_stats[~data_stats.job.str.contains("-iterative-")]
         data_stats = data_stats[["job", "build", "start_time", "duration"]]
 
         data_stats = data_stats[~data_stats.job.str.contains("-iterative-")]
         data_stats = data_stats[["job", "build", "start_time", "duration"]]
 

-        data_time_period = \
-            (datetime.utcnow() - data_stats["start_time"].min()).days
-        if self._time_period > data_time_period:
-            self._time_period = data_time_period
-

         jobs = sorted(list(data_stats["job"].unique()))
         d_job_info = {
             "job": list(),
         jobs = sorted(list(data_stats["job"].unique()))
         d_job_info = {
             "job": list(),


@@ -126,7 +113,7 @@ class Layout:
             "lst_failed": list()
         }
         for job in jobs:
             "lst_failed": list()
         }
         for job in jobs:

-            df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
+            df_job = data_trending.loc[(data_trending["job"] == job)]

             builds = df_job["build"].unique()
             for build in builds:
                 df_build = df_job.loc[(df_job["build"] == build)]
             builds = df_job["build"].unique()
             for build in builds:
                 df_build = df_job.loc[(df_job["build"] == build)]


@@ -207,7 +194,11 @@ class Layout:
             "ri-ttypes-value": self._default["ttype"],
             "ri-cadences-value": self._default["cadence"],
             "dd-tbeds-value": self._default["tbed"],
             "ri-ttypes-value": self._default["ttype"],
             "ri-cadences-value": self._default["cadence"],
             "dd-tbeds-value": self._default["tbed"],

-            "al-job-children": self._default["job"]
+            "al-job-children": html.A(
+                self._default["job"],
+                href=f"{C.URL_JENKINS}{self._default['job']}",
+                target="_blank"
+            )

         }
 
         # Callbacks:
         }
 
         # Callbacks:


@@ -246,7 +237,7 @@ class Layout:
                             self._add_navbar()
                         ]
                     ),
                             self._add_navbar()
                         ]
                     ),

-                    dcc.Loading(
+                    dbc.Spinner(

                         dbc.Offcanvas(
                             class_name="w-50",
                             id="offcanvas-metadata",
                         dbc.Offcanvas(
                             class_name="w-50",
                             id="offcanvas-metadata",


@@ -265,6 +256,18 @@ class Layout:
                             self._add_ctrl_col(),
                             self._add_plotting_col()
                         ]
                             self._add_ctrl_col(),
                             self._add_plotting_col()
                         ]

+                    ),
+                    dbc.Offcanvas(
+                        class_name="w-75",
+                        id="offcanvas-documentation",
+                        title="Documentation",
+                        placement="end",
+                        is_open=False,
+                        children=html.Iframe(
+                            src=C.URL_DOC_TRENDING,
+                            width="100%",
+                            height="100%"
+                        )

                     )
                 ]
             )
                     )
                 ]
             )


@@ -290,14 +293,26 @@ class Layout:
         return dbc.NavbarSimple(
             id="navbarsimple-main",
             children=[
         return dbc.NavbarSimple(
             id="navbarsimple-main",
             children=[

-                dbc.NavItem(
-                    dbc.NavLink(
-                        C.STATS_TITLE,
-                        disabled=True,
-                        external_link=True,
-                        href="#"
-                    )
-                )
+                dbc.NavItem(dbc.NavLink(
+                    C.TREND_TITLE,
+                    external_link=True,
+                    href="/trending"
+                )),
+                dbc.NavItem(dbc.NavLink(
+                    C.NEWS_TITLE,
+                    external_link=True,
+                    href="/news"
+                )),
+                dbc.NavItem(dbc.NavLink(
+                    C.STATS_TITLE,
+                    active=True,
+                    external_link=True,
+                    href="/stats"
+                )),
+                dbc.NavItem(dbc.NavLink(
+                    "Documentation",
+                    id="btn-documentation",
+                ))

             ],
             brand=C.BRAND,
             brand_href="/",
             ],
             brand=C.BRAND,
             brand_href="/",


@@ -328,7 +343,7 @@ class Layout:
         return dbc.Col(
             id="col-plotting-area",
             children=[
         return dbc.Col(
             id="col-plotting-area",
             children=[

-                dcc.Loading(
+                dbc.Spinner(

                     children=[
                         dbc.Row(
                             id="plotting-area",
                     children=[
                         dbc.Row(
                             id="plotting-area",


@@ -568,8 +583,8 @@ class Layout:
             Input("dd-tbeds", "value"),
             Input("url", "href")
         )
             Input("dd-tbeds", "value"),
             Input("url", "href")
         )

-        def _update_ctrl_panel(cp_data: dict, dut: str, ttype: str, cadence:str,
-                tbed: str, href: str) -> tuple:
+        def _update_ctrl_panel(cp_data: dict, dut: str, ttype: str,
+                cadence: str, tbed: str, href: str) -> tuple:

             """Update the application when the event is detected.
 
             :param cp_data: Current status of the control panel stored in
             """Update the application when the event is detected.
 
             :param cp_data: Current status of the control panel stored in


@@ -660,7 +675,14 @@ class Layout:
                                 "ri-ttypes-value": job_params["ttype"],
                                 "ri-cadences-value": job_params["cadence"],
                                 "dd-tbeds-value": job_params["tbed"],
                                 "ri-ttypes-value": job_params["ttype"],
                                 "ri-cadences-value": job_params["cadence"],
                                 "dd-tbeds-value": job_params["tbed"],

-                                "al-job-children": job_params["job"]
+                                "al-job-children": html.A(
+                                    self._default["job"],
+                                    href=(
+                                        f"{C.URL_JENKINS}"
+                                        f"{self._default['job']}"
+                                    ),
+                                    target="_blank"
+                                )

                             },
                             None
                         )
                             },
                             None
                         )


@@ -675,7 +697,15 @@ class Layout:
                 ctrl_panel.get("dd-tbeds-value")
             )
 
                 ctrl_panel.get("dd-tbeds-value")
             )
 

-            ctrl_panel.set({"al-job-children": job})
+            ctrl_panel.set(
+                {
+                    "al-job-children": html.A(
+                        job,
+                        href=f"{C.URL_JENKINS}{job}",
+                        target="_blank"
+                    )
+                }
+            )

             plotting_area = self._get_plotting_area(
                 job,
                 gen_new_url(parsed_url, {"job": job})
             plotting_area = self._get_plotting_area(
                 job,
                 gen_new_url(parsed_url, {"job": job})


@@ -793,11 +823,29 @@ class Layout:
                     fail_tests = None
 
                 # Create the content of the offcanvas:
                     fail_tests = None
 
                 # Create the content of the offcanvas:

+                list_group_items = list()
+                for itm in lst_graph_data:
+                    lst_itm = itm.split(": ")
+                    if lst_itm[0] == "csit-ref":
+                        list_group_item = dbc.ListGroupItem([
+                            dbc.Badge(lst_itm[0]),
+                            html.A(
+                                lst_itm[1],
+                                href=f"{C.URL_JENKINS}{lst_itm[1]}",
+                                target="_blank"
+                            )
+                        ])
+                    else:
+                        list_group_item = dbc.ListGroupItem([
+                            dbc.Badge(lst_itm[0]),
+                            lst_itm[1]
+                        ])
+                    list_group_items.append(list_group_item)

                 metadata = [
                     dbc.Card(
                         class_name="gy-2 p-0",
                         children=[
                 metadata = [
                     dbc.Card(
                         class_name="gy-2 p-0",
                         children=[

-                            dbc.CardHeader(children=[
+                            dbc.CardHeader([

                                 dcc.Clipboard(
                                     target_id="metadata",
                                     title="Copy",
                                 dcc.Clipboard(
                                     target_id="metadata",
                                     title="Copy",


@@ -806,21 +854,9 @@ class Layout:
                                 title
                             ]),
                             dbc.CardBody(
                                 title
                             ]),
                             dbc.CardBody(

+                                dbc.ListGroup(list_group_items, flush=True),

                                 id="metadata",
                                 id="metadata",

-                                class_name="p-0",
-                                children=[dbc.ListGroup(
-                                    children=[
-                                        dbc.ListGroupItem(
-                                            [
-                                                dbc.Badge(
-                                                    x.split(":")[0]
-                                                ),
-                                                x.split(": ")[1]
-                                            ]
-                                        ) for x in lst_graph_data
-                                    ],
-                                    flush=True),
-                                ]
+                                class_name="p-0"

                             )
                         ]
                     )
                             )
                         ]
                     )


@@ -852,3 +888,13 @@ class Layout:
                 open_canvas = True
 
             return metadata, open_canvas
                 open_canvas = True
 
             return metadata, open_canvas

+
+        @app.callback(
+            Output("offcanvas-documentation", "is_open"),
+            Input("btn-documentation", "n_clicks"),
+            State("offcanvas-documentation", "is_open")
+        )
+        def toggle_offcanvas_documentation(n_clicks, is_open):
+            if n_clicks:
+                return not is_open
+            return is_open