C-Dash: Fix anomaly detection for the news

[csit.git] / csit.infra.dash / app / cdash / news / layout.py

diff --git a/csit.infra.dash/app/cdash/news/layout.py b/csit.infra.dash/app/cdash/news/layout.py
index 4032f11..d8ad92a 100644 (file)
--- a/csit.infra.dash/app/cdash/news/layout.py
+++ b/csit.infra.dash/app/cdash/news/layout.py
@@ -14,7 +14,6 @@
 """Plotly Dash HTML layout override.
 """
 
-import logging
 import pandas as pd
 import dash_bootstrap_components as dbc
 
@@ -23,13 +22,11 @@ from dash import dcc
 from dash import html
 from dash import callback_context
 from dash import Input, Output, State
-from yaml import load, FullLoader, YAMLError
 
-from ..data.data import Data
 from ..utils.constants import Constants as C
-from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url
+from ..utils.utils import gen_new_url
+from ..utils.anomalies import classify_anomalies
 from ..utils.url_processing import url_decode
-from ..data.data import Data
 from .tables import table_summary
 
 
@@ -37,8 +34,13 @@ class Layout:
     """The layout of the dash app and the callbacks.
     """
 
-    def __init__(self, app: Flask, html_layout_file: str, data_spec_file: str,
-        tooltip_file: str) -> None:
+    def __init__(
+            self,
+            app: Flask,
+            data_stats: pd.DataFrame,
+            data_trending: pd.DataFrame,
+            html_layout_file: str
+        ) -> None:
         """Initialization:
         - save the input parameters,
         - read and pre-process the data,
@@ -47,34 +49,22 @@ class Layout:
         - read tooltips from the tooltip file.
 
         :param app: Flask application running the dash application.
+        :param data_stats: Pandas dataframe with staistical data.
+        :param data_trending: Pandas dataframe with trending data.
         :param html_layout_file: Path and name of the file specifying the HTML
             layout of the dash application.
-        :param data_spec_file: Path and name of the file specifying the data to
-            be read from parquets for this application.
-        :param tooltip_file: Path and name of the yaml file specifying the
-            tooltips.
         :type app: Flask
+        :type data_stats: pandas.DataFrame
+        :type data_trending: pandas.DataFrame
         :type html_layout_file: str
-        :type data_spec_file: str
-        :type tooltip_file: str
         """
 
         # Inputs
         self._app = app
         self._html_layout_file = html_layout_file
-        self._data_spec_file = data_spec_file
-        self._tooltip_file = tooltip_file
-
-        # Read the data:
-        data_stats, data_mrr, data_ndrpdr = Data(
-            data_spec_file=self._data_spec_file,
-            debug=True
-        ).read_stats(days=C.NEWS_TIME_PERIOD)
-
-        df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
 
         # Prepare information for the control panel:
-        self._jobs = sorted(list(df_tst_info["job"].unique()))
+        self._jobs = sorted(list(data_trending["job"].unique()))
         d_job_info = {
             "job": list(),
             "dut": list(),
@@ -115,7 +105,7 @@ class Layout:
         }
         for job in self._jobs:
             # Create lists of failed tests:
-            df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
+            df_job = data_trending.loc[(data_trending["job"] == job)]
             last_build = str(max(pd.to_numeric(df_job["build"].unique())))
             df_build = df_job.loc[(df_job["build"] == last_build)]
             tst_info["job"].append(job)
@@ -143,15 +133,17 @@ class Layout:
 
             tests = df_job["test_id"].unique()
             for test in tests:
-                tst_data = df_job.loc[df_job["test_id"] == test].sort_values(
-                    by="start_time", ignore_index=True)
-                x_axis = tst_data["start_time"].tolist()
+                tst_data = df_job.loc[(
+                    (df_job["test_id"] == test) &
+                    (df_job["passed"] == True)
+                )].sort_values(by="start_time", ignore_index=True)
                 if "-ndrpdr" in test:
                     tst_data = tst_data.dropna(
                         subset=["result_pdr_lower_rate_value", ]
                     )
                     if tst_data.empty:
                         continue
+                    x_axis = tst_data["start_time"].tolist()
                     try:
                         anomalies, _, _ = classify_anomalies({
                             k: v for k, v in zip(
@@ -196,6 +188,7 @@ class Layout:
                     )
                     if tst_data.empty:
                         continue
+                    x_axis = tst_data["start_time"].tolist()
                     try:
                         anomalies, _, _ = classify_anomalies({
                             k: v for k, v in zip(
@@ -226,7 +219,6 @@ class Layout:
 
         # Read from files:
         self._html_layout = str()
-        self._tooltips = dict()
 
         try:
             with open(self._html_layout_file, "r") as file_read:
@@ -236,19 +228,6 @@ class Layout:
                 f"Not possible to open the file {self._html_layout_file}\n{err}"
             )
 
-        try:
-            with open(self._tooltip_file, "r") as file_read:
-                self._tooltips = load(file_read, Loader=FullLoader)
-        except IOError as err:
-            logging.warning(
-                f"Not possible to open the file {self._tooltip_file}\n{err}"
-            )
-        except YAMLError as err:
-            logging.warning(
-                f"An error occurred while parsing the specification file "
-                f"{self._tooltip_file}\n{err}"
-            )
-
         self._default_period = C.NEWS_SHORT
         self._default_active = (False, True, False)
 
@@ -358,7 +337,7 @@ class Layout:
         return dbc.Col(
             id="col-plotting-area",
             children=[
-                dcc.Loading(
+                dbc.Spinner(
                     children=[
                         dbc.Row(
                             id="plotting-area",