X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fdash%2Fapp%2Fpal%2Ftrending%2Fgraphs.py;h=3b81cf39c4551cbc70723768605ecc6b2f23e2e6;hp=0760d9cc80e0352cfa4f50c29dd24658ee60402f;hb=650d20f1fc6bdea669982f2a549744fcdcce5a37;hpb=d6021416a08d5004bad5dd5220f53b6f1ecdf033

diff --git a/resources/tools/dash/app/pal/trending/graphs.py b/resources/tools/dash/app/pal/trending/graphs.py
index 0760d9cc80..3b81cf39c4 100644
--- a/resources/tools/dash/app/pal/trending/graphs.py
+++ b/resources/tools/dash/app/pal/trending/graphs.py
@@ -158,187 +158,209 @@ def _classify_anomalies(data):
     return classification, avgs, stdevs
 
 
-def graph_trending_tput(data: pd.DataFrame, sel:dict, layout: dict,
-    start: datetime, end: datetime) -> tuple:
+def select_trending_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
     """
     """
 
-    if not sel:
-        return None, None
+    phy = itm["phy"].split("-")
+    if len(phy) == 4:
+        topo, arch, nic, drv = phy
+        if drv == "dpdk":
+            drv = ""
+        else:
+            drv += "-"
+            drv = drv.replace("_", "-")
+    else:
+        return None
+
+    core = str() if itm["dut"] == "trex" else f"{itm['core']}"
+    ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
+    dut = "none" if itm["dut"] == "trex" else itm["dut"].upper()
 
-    def _generate_traces(ttype: str, name: str, df: pd.DataFrame,
-        start: datetime, end: datetime, color: str) -> list:
+    df = data.loc[(
+        (data["dut_type"] == dut) &
+        (data["test_type"] == ttype) &
+        (data["passed"] == True)
+    )]
+    df = df[df.job.str.endswith(f"{topo}-{arch}")]
+    df = df[df.test_id.str.contains(
+        f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
+        regex=True
+    )].sort_values(by="start_time", ignore_index=True)
 
-        df = df.dropna(subset=[_VALUE[ttype], ])
-        if df.empty:
-            return list()
+    return df
 
-        x_axis = [d for d in df["start_time"] if d >= start and d <= end]
 
-        anomalies, trend_avg, trend_stdev = _classify_anomalies(
-            {k: v for k, v in zip(x_axis, df[_VALUE[ttype]])}
+def _generate_trending_traces(ttype: str, name: str, df: pd.DataFrame,
+    start: datetime, end: datetime, color: str) -> list:
+    """
+    """
+
+    df = df.dropna(subset=[_VALUE[ttype], ])
+    if df.empty:
+        return list()
+    df = df.loc[((df["start_time"] >= start) & (df["start_time"] <= end))]
+    if df.empty:
+        return list()
+
+    x_axis = df["start_time"].tolist()
+
+    anomalies, trend_avg, trend_stdev = _classify_anomalies(
+        {k: v for k, v in zip(x_axis, df[_VALUE[ttype]])}
+    )
+
+    hover = list()
+    customdata = list()
+    for _, row in df.iterrows():
+        hover_itm = (
+            f"date: {row['start_time'].strftime('%d-%m-%Y %H:%M:%S')}<br>"
+            f"<prop> [{row[_UNIT[ttype]]}]: {row[_VALUE[ttype]]}<br>"
+            f"<stdev>"
+            f"{row['dut_type']}-ref: {row['dut_version']}<br>"
+            f"csit-ref: {row['job']}/{row['build']}<br>"
+            f"hosts: {', '.join(row['hosts'])}"
+        )
+        if ttype == "mrr":
+            stdev = (
+                f"stdev [{row['result_receive_rate_rate_unit']}]: "
+                f"{row['result_receive_rate_rate_stdev']}<br>"
+            )
+        else:
+            stdev = ""
+        hover_itm = hover_itm.replace(
+            "<prop>", "latency" if ttype == "pdr-lat" else "average"
+        ).replace("<stdev>", stdev)
+        hover.append(hover_itm)
+        if ttype == "pdr-lat":
+            customdata.append(_get_hdrh_latencies(row, name))
+
+    hover_trend = list()
+    for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
+        hover_itm = (
+            f"date: {row['start_time'].strftime('%d-%m-%Y %H:%M:%S')}<br>"
+            f"trend [pps]: {avg}<br>"
+            f"stdev [pps]: {stdev}<br>"
+            f"{row['dut_type']}-ref: {row['dut_version']}<br>"
+            f"csit-ref: {row['job']}/{row['build']}<br>"
+            f"hosts: {', '.join(row['hosts'])}"
         )
+        if ttype == "pdr-lat":
+            hover_itm = hover_itm.replace("[pps]", "[us]")
+        hover_trend.append(hover_itm)
+
+    traces = [
+        go.Scatter(  # Samples
+            x=x_axis,
+            y=df[_VALUE[ttype]],
+            name=name,
+            mode="markers",
+            marker={
+                u"size": 5,
+                u"color": color,
+                u"symbol": u"circle",
+            },
+            text=hover,
+            hoverinfo=u"text+name",
+            showlegend=True,
+            legendgroup=name,
+            customdata=customdata
+        ),
+        go.Scatter(  # Trend line
+            x=x_axis,
+            y=trend_avg,
+            name=name,
+            mode="lines",
+            line={
+                u"shape": u"linear",
+                u"width": 1,
+                u"color": color,
+            },
+            text=hover_trend,
+            hoverinfo=u"text+name",
+            showlegend=False,
+            legendgroup=name,
+        )
+    ]
 
+    if anomalies:
+        anomaly_x = list()
+        anomaly_y = list()
+        anomaly_color = list()
         hover = list()
-        customdata = list()
-        for _, row in df.iterrows():
-            hover_itm = (
-                f"date: {row['start_time'].strftime('%d-%m-%Y %H:%M:%S')}<br>"
-                f"<prop> [{row[_UNIT[ttype]]}]: {row[_VALUE[ttype]]}<br>"
-                f"<stdev>"
-                f"{row['dut_type']}-ref: {row['dut_version']}<br>"
-                f"csit-ref: {row['job']}/{row['build']}"
-            )
-            if ttype == "mrr":
-                stdev = (
-                    f"stdev [{row['result_receive_rate_rate_unit']}]: "
-                    f"{row['result_receive_rate_rate_stdev']}<br>"
-                )
-            else:
-                stdev = ""
-            hover_itm = hover_itm.replace(
-                "<prop>", "latency" if ttype == "pdr-lat" else "average"
-            ).replace("<stdev>", stdev)
-            hover.append(hover_itm)
-            if ttype == "pdr-lat":
-                customdata.append(_get_hdrh_latencies(row, name))
-
-        hover_trend = list()
-        for avg, stdev in zip(trend_avg, trend_stdev):
-            if ttype == "pdr-lat":
-                hover_trend.append(
-                    f"trend [us]: {avg}<br>"
-                    f"stdev [us]: {stdev}"
+        for idx, anomaly in enumerate(anomalies):
+            if anomaly in (u"regression", u"progression"):
+                anomaly_x.append(x_axis[idx])
+                anomaly_y.append(trend_avg[idx])
+                anomaly_color.append(_ANOMALY_COLOR[anomaly])
+                hover_itm = (
+                    f"date: {x_axis[idx].strftime('%d-%m-%Y %H:%M:%S')}<br>"
+                    f"trend [pps]: {trend_avg[idx]}<br>"
+                    f"classification: {anomaly}"
                 )
-            else:
-                hover_trend.append(
-                    f"trend [pps]: {avg}<br>"
-                    f"stdev [pps]: {stdev}"
-                )
-
-        traces = [
-            go.Scatter(  # Samples
-                x=x_axis,
-                y=df[_VALUE[ttype]],
-                name=name,
-                mode="markers",
-                marker={
-                    u"size": 5,
-                    u"color": color,
-                    u"symbol": u"circle",
-                },
+                if ttype == "pdr-lat":
+                    hover_itm = hover_itm.replace("[pps]", "[us]")
+                hover.append(hover_itm)
+        anomaly_color.extend([0.0, 0.5, 1.0])
+        traces.append(
+            go.Scatter(
+                x=anomaly_x,
+                y=anomaly_y,
+                mode=u"markers",
                 text=hover,
                 hoverinfo=u"text+name",
-                showlegend=True,
-                legendgroup=name,
-                customdata=customdata
-            ),
-            go.Scatter(  # Trend line
-                x=x_axis,
-                y=trend_avg,
-                name=name,
-                mode="lines",
-                line={
-                    u"shape": u"linear",
-                    u"width": 1,
-                    u"color": color,
-                },
-                text=hover_trend,
-                hoverinfo=u"text+name",
                 showlegend=False,
                 legendgroup=name,
-            )
-        ]
-
-        if anomalies:
-            anomaly_x = list()
-            anomaly_y = list()
-            anomaly_color = list()
-            for idx, anomaly in enumerate(anomalies):
-                if anomaly in (u"regression", u"progression"):
-                    anomaly_x.append(x_axis[idx])
-                    anomaly_y.append(trend_avg[idx])
-                    anomaly_color.append(_ANOMALY_COLOR[anomaly])
-            anomaly_color.extend([0.0, 0.5, 1.0])
-            traces.append(
-                go.Scatter(
-                    x=anomaly_x,
-                    y=anomaly_y,
-                    mode=u"markers",
-                    hoverinfo=u"none",
-                    showlegend=False,
-                    legendgroup=name,
-                    name=f"{name}-anomalies",
-                    marker={
-                        u"size": 15,
-                        u"symbol": u"circle-open",
-                        u"color": anomaly_color,
-                        u"colorscale": _COLORSCALE_LAT \
-                            if ttype == "pdr-lat" else _COLORSCALE_TPUT,
-                        u"showscale": True,
-                        u"line": {
-                            u"width": 2
-                        },
-                        u"colorbar": {
-                            u"y": 0.5,
-                            u"len": 0.8,
-                            u"title": u"Circles Marking Data Classification",
-                            u"titleside": u"right",
-                            # u"titlefont": {
-                            #     u"size": 14
-                            # },
-                            u"tickmode": u"array",
-                            u"tickvals": [0.167, 0.500, 0.833],
-                            u"ticktext": _TICK_TEXT_LAT \
-                                if ttype == "pdr-lat" else _TICK_TEXT_TPUT,
-                            u"ticks": u"",
-                            u"ticklen": 0,
-                            u"tickangle": -90,
-                            u"thickness": 10
-                        }
+                name=name,
+                marker={
+                    u"size": 15,
+                    u"symbol": u"circle-open",
+                    u"color": anomaly_color,
+                    u"colorscale": _COLORSCALE_LAT \
+                        if ttype == "pdr-lat" else _COLORSCALE_TPUT,
+                    u"showscale": True,
+                    u"line": {
+                        u"width": 2
+                    },
+                    u"colorbar": {
+                        u"y": 0.5,
+                        u"len": 0.8,
+                        u"title": u"Circles Marking Data Classification",
+                        u"titleside": u"right",
+                        u"tickmode": u"array",
+                        u"tickvals": [0.167, 0.500, 0.833],
+                        u"ticktext": _TICK_TEXT_LAT \
+                            if ttype == "pdr-lat" else _TICK_TEXT_TPUT,
+                        u"ticks": u"",
+                        u"ticklen": 0,
+                        u"tickangle": -90,
+                        u"thickness": 10
                     }
-                )
+                }
             )
+        )
+
+    return traces
+
+
+def graph_trending(data: pd.DataFrame, sel:dict, layout: dict,
+    start: datetime, end: datetime) -> tuple:
+    """
+    """
 
-        return traces
+    if not sel:
+        return None, None
 
-    # Generate graph:
     fig_tput = None
     fig_lat = None
     for idx, itm in enumerate(sel):
-        phy = itm["phy"].split("-")
-        if len(phy) == 4:
-            topo, arch, nic, drv = phy
-            if drv in ("dpdk", "ixgbe"):
-                drv = ""
-            else:
-                drv += "-"
-                drv = drv.replace("_", "-")
-        else:
+
+        df = select_trending_data(data, itm)
+        if df is None or df.empty:
             continue
-        cadence = \
-            "weekly" if (arch == "aws" or itm["testtype"] != "mrr") else "daily"
-        sel_topo_arch = (
-            f"csit-vpp-perf-"
-            f"{itm['testtype'] if itm['testtype'] == 'mrr' else 'ndrpdr'}-"
-            f"{cadence}-master-{topo}-{arch}"
-        )
-        df_sel = data.loc[(data["job"] == sel_topo_arch)]
-        regex = (
-            f"^.*{nic}.*\.{itm['framesize']}-{itm['core']}-{drv}{itm['test']}-"
-            f"{'mrr' if itm['testtype'] == 'mrr' else 'ndrpdr'}$"
-        )
-        df = df_sel.loc[
-            df_sel["test_id"].apply(
-                lambda x: True if re.search(regex, x) else False
-            )
-        ].sort_values(by="start_time", ignore_index=True)
-        name = (
-            f"{itm['phy']}-{itm['framesize']}-{itm['core']}-"
-            f"{itm['test']}-{itm['testtype']}"
-        )
 
-        traces = _generate_traces(
+        name = "-".join((itm["dut"], itm["phy"], itm["framesize"], itm["core"],
+            itm["test"], itm["testtype"], ))
+        traces = _generate_trending_traces(
             itm["testtype"], name, df, start, end, _COLORS[idx % len(_COLORS)]
         )
         if traces:
@@ -347,7 +369,7 @@ def graph_trending_tput(data: pd.DataFrame, sel:dict, layout: dict,
             fig_tput.add_traces(traces)
 
         if itm["testtype"] == "pdr":
-            traces = _generate_traces(
+            traces = _generate_trending_traces(
                 "pdr-lat", name, df, start, end, _COLORS[idx % len(_COLORS)]
             )
             if traces:
@@ -369,11 +391,6 @@ def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 
     fig = None
 
-    try:
-        name = data.pop("name")
-    except (KeyError, AttributeError):
-        return None
-
     traces = list()
     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
         try:
@@ -433,7 +450,6 @@ def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
         fig.add_traces(traces)
         layout_hdrh = layout.get("plot-hdrh-latency", None)
         if lat_hdrh:
-            layout_hdrh["title"]["text"] = f"<b>{name}</b>"
             fig.update_layout(layout_hdrh)
 
     return fig