csit.infra.dash/app/cdash/trending/graphs.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """
  15 """
  16
  17 import plotly.graph_objects as go
  18 import pandas as pd
  19
  20 import hdrh.histogram
  21 import hdrh.codec
  22
  23 from ..utils.constants import Constants as C
  24 from ..utils.utils import classify_anomalies, get_color
  25
  26
  27 def _get_hdrh_latencies(row: pd.Series, name: str) -> dict:
  28     """Get the HDRH latencies from the test data.
  29
  30     :param row: A row fron the data frame with test data.
  31     :param name: The test name to be displayed as the graph title.
  32     :type row: pandas.Series
  33     :type name: str
  34     :returns: Dictionary with HDRH latencies.
  35     :rtype: dict
  36     """
  37
  38     latencies = {"name": name}
  39     for key in C.LAT_HDRH:
  40         try:
  41             latencies[key] = row[key]
  42         except KeyError:
  43             return None
  44
  45     return latencies
  46
  47
  48 def select_trending_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
  49     """Select the data for graphs from the provided data frame.
  50
  51     :param data: Data frame with data for graphs.
  52     :param itm: Item (in this case job name) which data will be selected from
  53         the input data frame.
  54     :type data: pandas.DataFrame
  55     :type itm: str
  56     :returns: A data frame with selected data.
  57     :rtype: pandas.DataFrame
  58     """
  59
  60     phy = itm["phy"].split("-")
  61     if len(phy) == 4:
  62         topo, arch, nic, drv = phy
  63         if drv == "dpdk":
  64             drv = ""
  65         else:
  66             drv += "-"
  67             drv = drv.replace("_", "-")
  68     else:
  69         return None
  70
  71     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
  72     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
  73
  74     df = data.loc[(
  75         (data["test_type"] == ttype) &
  76         (data["passed"] == True)
  77     )]
  78     df = df[df.job.str.endswith(f"{topo}-{arch}")]
  79     df = df[df.test_id.str.contains(
  80         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
  81         regex=True
  82     )].sort_values(by="start_time", ignore_index=True)
  83
  84     return df
  85
  86
  87 def _generate_trending_traces(ttype: str, name: str, df: pd.DataFrame,
  88     color: str, norm_factor: float) -> list:
  89     """Generate the trending traces for the trending graph.
  90
  91     :param ttype: Test type (MRR, NDR, PDR).
  92     :param name: The test name to be displayed as the graph title.
  93     :param df: Data frame with test data.
  94     :param color: The color of the trace (samples and trend line).
  95     :param norm_factor: The factor used for normalization of the results to CPU
  96         frequency set to Constants.NORM_FREQUENCY.
  97     :type ttype: str
  98     :type name: str
  99     :type df: pandas.DataFrame
 100     :type color: str
 101     :type norm_factor: float
 102     :returns: Traces (samples, trending line, anomalies)
 103     :rtype: list
 104     """
 105
 106     df = df.dropna(subset=[C.VALUE[ttype], ])
 107     if df.empty:
 108         return list()
 109
 110     x_axis = df["start_time"].tolist()
 111     if ttype == "pdr-lat":
 112         y_data = [(itm / norm_factor) for itm in df[C.VALUE[ttype]].tolist()]
 113     else:
 114         y_data = [(itm * norm_factor) for itm in df[C.VALUE[ttype]].tolist()]
 115
 116     anomalies, trend_avg, trend_stdev = classify_anomalies(
 117         {k: v for k, v in zip(x_axis, y_data)}
 118     )
 119
 120     hover = list()
 121     customdata = list()
 122     customdata_samples = list()
 123     for idx, (_, row) in enumerate(df.iterrows()):
 124         d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 125         hover_itm = (
 126             f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 127             f"<prop> [{row[C.UNIT[ttype]]}]: {y_data[idx]:,.0f}<br>"
 128             f"<stdev>"
 129             f"{d_type}-ref: {row['dut_version']}<br>"
 130             f"csit-ref: {row['job']}/{row['build']}<br>"
 131             f"hosts: {', '.join(row['hosts'])}"
 132         )
 133         if ttype == "mrr":
 134             stdev = (
 135                 f"stdev [{row['result_receive_rate_rate_unit']}]: "
 136                 f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 137             )
 138         else:
 139             stdev = ""
 140         hover_itm = hover_itm.replace(
 141             "<prop>", "latency" if ttype == "pdr-lat" else "average"
 142         ).replace("<stdev>", stdev)
 143         hover.append(hover_itm)
 144         if ttype == "pdr-lat":
 145             customdata_samples.append(_get_hdrh_latencies(row, name))
 146             customdata.append({"name": name})
 147         else:
 148             customdata_samples.append({"name": name, "show_telemetry": True})
 149             customdata.append({"name": name})
 150
 151     hover_trend = list()
 152     for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 153         d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 154         hover_itm = (
 155             f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 156             f"trend [pps]: {avg:,.0f}<br>"
 157             f"stdev [pps]: {stdev:,.0f}<br>"
 158             f"{d_type}-ref: {row['dut_version']}<br>"
 159             f"csit-ref: {row['job']}/{row['build']}<br>"
 160             f"hosts: {', '.join(row['hosts'])}"
 161         )
 162         if ttype == "pdr-lat":
 163             hover_itm = hover_itm.replace("[pps]", "[us]")
 164         hover_trend.append(hover_itm)
 165
 166     traces = [
 167         go.Scatter(  # Samples
 168             x=x_axis,
 169             y=y_data,
 170             name=name,
 171             mode="markers",
 172             marker={
 173                 "size": 5,
 174                 "color": color,
 175                 "symbol": "circle",
 176             },
 177             text=hover,
 178             hoverinfo="text+name",
 179             showlegend=True,
 180             legendgroup=name,
 181             customdata=customdata_samples
 182         ),
 183         go.Scatter(  # Trend line
 184             x=x_axis,
 185             y=trend_avg,
 186             name=name,
 187             mode="lines",
 188             line={
 189                 "shape": "linear",
 190                 "width": 1,
 191                 "color": color,
 192             },
 193             text=hover_trend,
 194             hoverinfo="text+name",
 195             showlegend=False,
 196             legendgroup=name,
 197             customdata=customdata
 198         )
 199     ]
 200
 201     if anomalies:
 202         anomaly_x = list()
 203         anomaly_y = list()
 204         anomaly_color = list()
 205         hover = list()
 206         for idx, anomaly in enumerate(anomalies):
 207             if anomaly in ("regression", "progression"):
 208                 anomaly_x.append(x_axis[idx])
 209                 anomaly_y.append(trend_avg[idx])
 210                 anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 211                 hover_itm = (
 212                     f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 213                     f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
 214                     f"classification: {anomaly}"
 215                 )
 216                 if ttype == "pdr-lat":
 217                     hover_itm = hover_itm.replace("[pps]", "[us]")
 218                 hover.append(hover_itm)
 219         anomaly_color.extend([0.0, 0.5, 1.0])
 220         traces.append(
 221             go.Scatter(
 222                 x=anomaly_x,
 223                 y=anomaly_y,
 224                 mode="markers",
 225                 text=hover,
 226                 hoverinfo="text+name",
 227                 showlegend=False,
 228                 legendgroup=name,
 229                 name=name,
 230                 customdata=customdata,
 231                 marker={
 232                     "size": 15,
 233                     "symbol": "circle-open",
 234                     "color": anomaly_color,
 235                     "colorscale": C.COLORSCALE_LAT \
 236                         if ttype == "pdr-lat" else C.COLORSCALE_TPUT,
 237                     "showscale": True,
 238                     "line": {
 239                         "width": 2
 240                     },
 241                     "colorbar": {
 242                         "y": 0.5,
 243                         "len": 0.8,
 244                         "title": "Circles Marking Data Classification",
 245                         "titleside": "right",
 246                         "tickmode": "array",
 247                         "tickvals": [0.167, 0.500, 0.833],
 248                         "ticktext": C.TICK_TEXT_LAT \
 249                             if ttype == "pdr-lat" else C.TICK_TEXT_TPUT,
 250                         "ticks": "",
 251                         "ticklen": 0,
 252                         "tickangle": -90,
 253                         "thickness": 10
 254                     }
 255                 }
 256             )
 257         )
 258
 259     return traces
 260
 261
 262 def graph_trending(data: pd.DataFrame, sel:dict, layout: dict,
 263     normalize: bool) -> tuple:
 264     """Generate the trending graph(s) - MRR, NDR, PDR and for PDR also Latences
 265     (result_latency_forward_pdr_50_avg).
 266
 267     :param data: Data frame with test results.
 268     :param sel: Selected tests.
 269     :param layout: Layout of plot.ly graph.
 270     :param normalize: If True, the data is normalized to CPU frquency
 271         Constants.NORM_FREQUENCY.
 272     :type data: pandas.DataFrame
 273     :type sel: dict
 274     :type layout: dict
 275     :type normalize: bool
 276     :returns: Trending graph(s)
 277     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
 278     """
 279
 280     if not sel:
 281         return None, None
 282
 283     fig_tput = None
 284     fig_lat = None
 285     for idx, itm in enumerate(sel):
 286
 287         df = select_trending_data(data, itm)
 288         if df is None or df.empty:
 289             continue
 290
 291         if normalize:
 292             phy = itm["phy"].split("-")
 293             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 294             norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY[topo_arch]) \
 295                 if topo_arch else 1.0
 296         else:
 297             norm_factor = 1.0
 298         traces = _generate_trending_traces(itm["testtype"], itm["id"], df,
 299             get_color(idx), norm_factor)
 300         if traces:
 301             if not fig_tput:
 302                 fig_tput = go.Figure()
 303             fig_tput.add_traces(traces)
 304
 305         if itm["testtype"] == "pdr":
 306             traces = _generate_trending_traces("pdr-lat", itm["id"], df,
 307                 get_color(idx), norm_factor)
 308             if traces:
 309                 if not fig_lat:
 310                     fig_lat = go.Figure()
 311                 fig_lat.add_traces(traces)
 312
 313     if fig_tput:
 314         fig_tput.update_layout(layout.get("plot-trending-tput", dict()))
 315     if fig_lat:
 316         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 317
 318     return fig_tput, fig_lat
 319
 320
 321 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 322     """Generate HDR Latency histogram graphs.
 323
 324     :param data: HDRH data.
 325     :param layout: Layout of plot.ly graph.
 326     :type data: dict
 327     :type layout: dict
 328     :returns: HDR latency Histogram.
 329     :rtype: plotly.graph_objects.Figure
 330     """
 331
 332     fig = None
 333
 334     traces = list()
 335     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
 336         try:
 337             decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
 338         except (hdrh.codec.HdrLengthException, TypeError):
 339             continue
 340         previous_x = 0.0
 341         prev_perc = 0.0
 342         xaxis = list()
 343         yaxis = list()
 344         hovertext = list()
 345         for item in decoded.get_recorded_iterator():
 346             # The real value is "percentile".
 347             # For 100%, we cut that down to "x_perc" to avoid
 348             # infinity.
 349             percentile = item.percentile_level_iterated_to
 350             x_perc = min(percentile, C.PERCENTILE_MAX)
 351             xaxis.append(previous_x)
 352             yaxis.append(item.value_iterated_to)
 353             hovertext.append(
 354                 f"<b>{C.GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 355                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 356                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 357                 f"Latency: {item.value_iterated_to}uSec"
 358             )
 359             next_x = 100.0 / (100.0 - x_perc)
 360             xaxis.append(next_x)
 361             yaxis.append(item.value_iterated_to)
 362             hovertext.append(
 363                 f"<b>{C.GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 364                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 365                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 366                 f"Latency: {item.value_iterated_to}uSec"
 367             )
 368             previous_x = next_x
 369             prev_perc = percentile
 370
 371         traces.append(
 372             go.Scatter(
 373                 x=xaxis,
 374                 y=yaxis,
 375                 name=C.GRAPH_LAT_HDRH_DESC[lat_name],
 376                 mode="lines",
 377                 legendgroup=C.GRAPH_LAT_HDRH_DESC[lat_name],
 378                 showlegend=bool(idx % 2),
 379                 line=dict(
 380                     color=get_color(int(idx/2)),
 381                     dash="solid",
 382                     width=1 if idx % 2 else 2
 383                 ),
 384                 hovertext=hovertext,
 385                 hoverinfo="text"
 386             )
 387         )
 388     if traces:
 389         fig = go.Figure()
 390         fig.add_traces(traces)
 391         layout_hdrh = layout.get("plot-hdrh-latency", None)
 392         if lat_hdrh:
 393             fig.update_layout(layout_hdrh)
 394
 395     return fig