csit.infra.dash/app/cdash/trending/graphs.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """
  15 """
  16
  17 import plotly.graph_objects as go
  18 import pandas as pd
  19
  20 import hdrh.histogram
  21 import hdrh.codec
  22
  23 from ..utils.constants import Constants as C
  24 from ..utils.utils import classify_anomalies, get_color
  25
  26
  27 def _get_hdrh_latencies(row: pd.Series, name: str) -> dict:
  28     """Get the HDRH latencies from the test data.
  29
  30     :param row: A row fron the data frame with test data.
  31     :param name: The test name to be displayed as the graph title.
  32     :type row: pandas.Series
  33     :type name: str
  34     :returns: Dictionary with HDRH latencies.
  35     :rtype: dict
  36     """
  37
  38     latencies = {"name": name}
  39     for key in C.LAT_HDRH:
  40         try:
  41             latencies[key] = row[key]
  42         except KeyError:
  43             return None
  44
  45     return latencies
  46
  47
  48 def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
  49     """Select the data for graphs from the provided data frame.
  50
  51     :param data: Data frame with data for graphs.
  52     :param itm: Item (in this case job name) which data will be selected from
  53         the input data frame.
  54     :type data: pandas.DataFrame
  55     :type itm: dict
  56     :returns: A data frame with selected data.
  57     :rtype: pandas.DataFrame
  58     """
  59
  60     phy = itm["phy"].split("-")
  61     if len(phy) == 4:
  62         topo, arch, nic, drv = phy
  63         if drv == "dpdk":
  64             drv = ""
  65         else:
  66             drv += "-"
  67             drv = drv.replace("_", "-")
  68     else:
  69         return None
  70
  71     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
  72     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
  73
  74     df = data.loc[(
  75         (data["test_type"] == ttype) &
  76         (data["passed"] == True)
  77     )]
  78     df = df[df.job.str.endswith(f"{topo}-{arch}")]
  79     df = df[df.test_id.str.contains(
  80         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
  81         regex=True
  82     )].sort_values(by="start_time", ignore_index=True)
  83
  84     return df
  85
  86
  87 def graph_trending(
  88         data: pd.DataFrame,
  89         sel: dict,
  90         layout: dict,
  91         normalize: bool
  92     ) -> tuple:
  93     """Generate the trending graph(s) - MRR, NDR, PDR and for PDR also Latences
  94     (result_latency_forward_pdr_50_avg).
  95
  96     :param data: Data frame with test results.
  97     :param sel: Selected tests.
  98     :param layout: Layout of plot.ly graph.
  99     :param normalize: If True, the data is normalized to CPU frquency
 100         Constants.NORM_FREQUENCY.
 101     :type data: pandas.DataFrame
 102     :type sel: dict
 103     :type layout: dict
 104     :type normalize: bool
 105     :returns: Trending graph(s)
 106     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
 107     """
 108
 109     if not sel:
 110         return None, None
 111
 112
 113     def _generate_trending_traces(
 114             ttype: str,
 115             name: str,
 116             df: pd.DataFrame,
 117             color: str,
 118             norm_factor: float
 119         ) -> list:
 120         """Generate the trending traces for the trending graph.
 121
 122         :param ttype: Test type (MRR, NDR, PDR).
 123         :param name: The test name to be displayed as the graph title.
 124         :param df: Data frame with test data.
 125         :param color: The color of the trace (samples and trend line).
 126         :param norm_factor: The factor used for normalization of the results to
 127             CPU frequency set to Constants.NORM_FREQUENCY.
 128         :type ttype: str
 129         :type name: str
 130         :type df: pandas.DataFrame
 131         :type color: str
 132         :type norm_factor: float
 133         :returns: Traces (samples, trending line, anomalies)
 134         :rtype: list
 135         """
 136
 137         df = df.dropna(subset=[C.VALUE[ttype], ])
 138         if df.empty:
 139             return list()
 140
 141         x_axis = df["start_time"].tolist()
 142         if ttype == "pdr-lat":
 143             y_data = [(v / norm_factor) for v in df[C.VALUE[ttype]].tolist()]
 144         else:
 145             y_data = [(v * norm_factor) for v in df[C.VALUE[ttype]].tolist()]
 146
 147         anomalies, trend_avg, trend_stdev = classify_anomalies(
 148             {k: v for k, v in zip(x_axis, y_data)}
 149         )
 150
 151         hover = list()
 152         customdata = list()
 153         customdata_samples = list()
 154         for idx, (_, row) in enumerate(df.iterrows()):
 155             d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 156             hover_itm = (
 157                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 158                 f"<prop> [{row[C.UNIT[ttype]]}]: {y_data[idx]:,.0f}<br>"
 159                 f"<stdev>"
 160                 f"{d_type}-ref: {row['dut_version']}<br>"
 161                 f"csit-ref: {row['job']}/{row['build']}<br>"
 162                 f"hosts: {', '.join(row['hosts'])}"
 163             )
 164             if ttype == "mrr":
 165                 stdev = (
 166                     f"stdev [{row['result_receive_rate_rate_unit']}]: "
 167                     f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 168                 )
 169             else:
 170                 stdev = ""
 171             hover_itm = hover_itm.replace(
 172                 "<prop>", "latency" if ttype == "pdr-lat" else "average"
 173             ).replace("<stdev>", stdev)
 174             hover.append(hover_itm)
 175             if ttype == "pdr-lat":
 176                 customdata_samples.append(_get_hdrh_latencies(row, name))
 177                 customdata.append({"name": name})
 178             else:
 179                 customdata_samples.append(
 180                     {"name": name, "show_telemetry": True}
 181                 )
 182                 customdata.append({"name": name})
 183
 184         hover_trend = list()
 185         for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 186             d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 187             hover_itm = (
 188                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 189                 f"trend [pps]: {avg:,.0f}<br>"
 190                 f"stdev [pps]: {stdev:,.0f}<br>"
 191                 f"{d_type}-ref: {row['dut_version']}<br>"
 192                 f"csit-ref: {row['job']}/{row['build']}<br>"
 193                 f"hosts: {', '.join(row['hosts'])}"
 194             )
 195             if ttype == "pdr-lat":
 196                 hover_itm = hover_itm.replace("[pps]", "[us]")
 197             hover_trend.append(hover_itm)
 198
 199         traces = [
 200             go.Scatter(  # Samples
 201                 x=x_axis,
 202                 y=y_data,
 203                 name=name,
 204                 mode="markers",
 205                 marker={
 206                     "size": 5,
 207                     "color": color,
 208                     "symbol": "circle",
 209                 },
 210                 text=hover,
 211                 hoverinfo="text+name",
 212                 showlegend=True,
 213                 legendgroup=name,
 214                 customdata=customdata_samples
 215             ),
 216             go.Scatter(  # Trend line
 217                 x=x_axis,
 218                 y=trend_avg,
 219                 name=name,
 220                 mode="lines",
 221                 line={
 222                     "shape": "linear",
 223                     "width": 1,
 224                     "color": color,
 225                 },
 226                 text=hover_trend,
 227                 hoverinfo="text+name",
 228                 showlegend=False,
 229                 legendgroup=name,
 230                 customdata=customdata
 231             )
 232         ]
 233
 234         if anomalies:
 235             anomaly_x = list()
 236             anomaly_y = list()
 237             anomaly_color = list()
 238             hover = list()
 239             for idx, anomaly in enumerate(anomalies):
 240                 if anomaly in ("regression", "progression"):
 241                     anomaly_x.append(x_axis[idx])
 242                     anomaly_y.append(trend_avg[idx])
 243                     anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 244                     hover_itm = (
 245                         f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 246                         f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
 247                         f"classification: {anomaly}"
 248                     )
 249                     if ttype == "pdr-lat":
 250                         hover_itm = hover_itm.replace("[pps]", "[us]")
 251                     hover.append(hover_itm)
 252             anomaly_color.extend([0.0, 0.5, 1.0])
 253             traces.append(
 254                 go.Scatter(
 255                     x=anomaly_x,
 256                     y=anomaly_y,
 257                     mode="markers",
 258                     text=hover,
 259                     hoverinfo="text+name",
 260                     showlegend=False,
 261                     legendgroup=name,
 262                     name=name,
 263                     customdata=customdata,
 264                     marker={
 265                         "size": 15,
 266                         "symbol": "circle-open",
 267                         "color": anomaly_color,
 268                         "colorscale": C.COLORSCALE_LAT \
 269                             if ttype == "pdr-lat" else C.COLORSCALE_TPUT,
 270                         "showscale": True,
 271                         "line": {
 272                             "width": 2
 273                         },
 274                         "colorbar": {
 275                             "y": 0.5,
 276                             "len": 0.8,
 277                             "title": "Circles Marking Data Classification",
 278                             "titleside": "right",
 279                             "tickmode": "array",
 280                             "tickvals": [0.167, 0.500, 0.833],
 281                             "ticktext": C.TICK_TEXT_LAT \
 282                                 if ttype == "pdr-lat" else C.TICK_TEXT_TPUT,
 283                             "ticks": "",
 284                             "ticklen": 0,
 285                             "tickangle": -90,
 286                             "thickness": 10
 287                         }
 288                     }
 289                 )
 290             )
 291
 292         return traces
 293
 294
 295     fig_tput = None
 296     fig_lat = None
 297     for idx, itm in enumerate(sel):
 298         df = select_trending_data(data, itm)
 299         if df is None or df.empty:
 300             continue
 301
 302         if normalize:
 303             phy = itm["phy"].split("-")
 304             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 305             norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY[topo_arch]) \
 306                 if topo_arch else 1.0
 307         else:
 308             norm_factor = 1.0
 309         traces = _generate_trending_traces(itm["testtype"], itm["id"], df,
 310             get_color(idx), norm_factor)
 311         if traces:
 312             if not fig_tput:
 313                 fig_tput = go.Figure()
 314             fig_tput.add_traces(traces)
 315
 316         if itm["testtype"] == "pdr":
 317             traces = _generate_trending_traces("pdr-lat", itm["id"], df,
 318                 get_color(idx), norm_factor)
 319             if traces:
 320                 if not fig_lat:
 321                     fig_lat = go.Figure()
 322                 fig_lat.add_traces(traces)
 323
 324     if fig_tput:
 325         fig_tput.update_layout(layout.get("plot-trending-tput", dict()))
 326     if fig_lat:
 327         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 328
 329     return fig_tput, fig_lat
 330
 331
 332 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 333     """Generate HDR Latency histogram graphs.
 334
 335     :param data: HDRH data.
 336     :param layout: Layout of plot.ly graph.
 337     :type data: dict
 338     :type layout: dict
 339     :returns: HDR latency Histogram.
 340     :rtype: plotly.graph_objects.Figure
 341     """
 342
 343     fig = None
 344
 345     traces = list()
 346     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
 347         try:
 348             decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
 349         except (hdrh.codec.HdrLengthException, TypeError):
 350             continue
 351         previous_x = 0.0
 352         prev_perc = 0.0
 353         xaxis = list()
 354         yaxis = list()
 355         hovertext = list()
 356         for item in decoded.get_recorded_iterator():
 357             # The real value is "percentile".
 358             # For 100%, we cut that down to "x_perc" to avoid
 359             # infinity.
 360             percentile = item.percentile_level_iterated_to
 361             x_perc = min(percentile, C.PERCENTILE_MAX)
 362             xaxis.append(previous_x)
 363             yaxis.append(item.value_iterated_to)
 364             hovertext.append(
 365                 f"<b>{C.GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 366                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 367                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 368                 f"Latency: {item.value_iterated_to}uSec"
 369             )
 370             next_x = 100.0 / (100.0 - x_perc)
 371             xaxis.append(next_x)
 372             yaxis.append(item.value_iterated_to)
 373             hovertext.append(
 374                 f"<b>{C.GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 375                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 376                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 377                 f"Latency: {item.value_iterated_to}uSec"
 378             )
 379             previous_x = next_x
 380             prev_perc = percentile
 381
 382         traces.append(
 383             go.Scatter(
 384                 x=xaxis,
 385                 y=yaxis,
 386                 name=C.GRAPH_LAT_HDRH_DESC[lat_name],
 387                 mode="lines",
 388                 legendgroup=C.GRAPH_LAT_HDRH_DESC[lat_name],
 389                 showlegend=bool(idx % 2),
 390                 line=dict(
 391                     color=get_color(int(idx/2)),
 392                     dash="solid",
 393                     width=1 if idx % 2 else 2
 394                 ),
 395                 hovertext=hovertext,
 396                 hoverinfo="text"
 397             )
 398         )
 399     if traces:
 400         fig = go.Figure()
 401         fig.add_traces(traces)
 402         layout_hdrh = layout.get("plot-hdrh-latency", None)
 403         if lat_hdrh:
 404             fig.update_layout(layout_hdrh)
 405
 406     return fig
 407
 408
 409 def graph_tm_trending(data: pd.DataFrame, layout: dict) -> list:
 410     """Generates one trending graph per test, each graph includes all selected
 411     metrics.
 412
 413     :param data: Data frame with telemetry data.
 414     :param layout: Layout of plot.ly graph.
 415     :type data: pandas.DataFrame
 416     :type layout: dict
 417     :returns: List of generated graphs together with test names.
 418         list(tuple(plotly.graph_objects.Figure(), str()), tuple(...), ...)
 419     :rtype: list
 420     """
 421
 422
 423     def _generate_graph(
 424             data: pd.DataFrame,
 425             test: str,
 426             layout: dict
 427         ) -> go.Figure:
 428         """Generates a trending graph for given test with all metrics.
 429
 430         :param data: Data frame with telemetry data for the given test.
 431         :param test: The name of the test.
 432         :param layout: Layout of plot.ly graph.
 433         :type data: pandas.DataFrame
 434         :type test: str
 435         :type layout: dict
 436         :returns: A trending graph.
 437         :rtype: plotly.graph_objects.Figure
 438         """
 439         graph = None
 440         traces = list()
 441         for idx, metric in enumerate(data.tm_metric.unique()):
 442             if "-pdr" in test and "='pdr'" not in metric:
 443                 continue
 444             if "-ndr" in test and "='ndr'" not in metric:
 445                 continue
 446
 447             df = data.loc[(data["tm_metric"] == metric)]
 448             x_axis = df["start_time"].tolist()
 449             y_data = [float(itm) for itm in df["tm_value"].tolist()]
 450             hover = list()
 451             for i, (_, row) in enumerate(df.iterrows()):
 452                 hover.append(
 453                     f"date: "
 454                     f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 455                     f"value: {y_data[i]:,.0f}<br>"
 456                     f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 457                     f"csit-ref: {row['job']}/{row['build']}<br>"
 458                 )
 459             if any(y_data):
 460                 anomalies, trend_avg, trend_stdev = classify_anomalies(
 461                     {k: v for k, v in zip(x_axis, y_data)}
 462                 )
 463                 hover_trend = list()
 464                 for avg, stdev, (_, row) in \
 465                         zip(trend_avg, trend_stdev, df.iterrows()):
 466                     hover_trend.append(
 467                         f"date: "
 468                         f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 469                         f"trend: {avg:,.0f}<br>"
 470                         f"stdev: {stdev:,.0f}<br>"
 471                         f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 472                         f"csit-ref: {row['job']}/{row['build']}"
 473                     )
 474             else:
 475                 anomalies = None
 476             color = get_color(idx)
 477             traces.append(
 478                 go.Scatter(  # Samples
 479                     x=x_axis,
 480                     y=y_data,
 481                     name=metric,
 482                     mode="markers",
 483                     marker={
 484                         "size": 5,
 485                         "color": color,
 486                         "symbol": "circle",
 487                     },
 488                     text=hover,
 489                     hoverinfo="text+name",
 490                     showlegend=True,
 491                     legendgroup=metric
 492                 )
 493             )
 494             if anomalies:
 495                 traces.append(
 496                     go.Scatter(  # Trend line
 497                         x=x_axis,
 498                         y=trend_avg,
 499                         name=metric,
 500                         mode="lines",
 501                         line={
 502                             "shape": "linear",
 503                             "width": 1,
 504                             "color": color,
 505                         },
 506                         text=hover_trend,
 507                         hoverinfo="text+name",
 508                         showlegend=False,
 509                         legendgroup=metric
 510                     )
 511                 )
 512
 513                 anomaly_x = list()
 514                 anomaly_y = list()
 515                 anomaly_color = list()
 516                 hover = list()
 517                 for idx, anomaly in enumerate(anomalies):
 518                     if anomaly in ("regression", "progression"):
 519                         anomaly_x.append(x_axis[idx])
 520                         anomaly_y.append(trend_avg[idx])
 521                         anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 522                         hover_itm = (
 523                             f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}"
 524                             f"<br>trend: {trend_avg[idx]:,.0f}"
 525                             f"<br>classification: {anomaly}"
 526                         )
 527                         hover.append(hover_itm)
 528                 anomaly_color.extend([0.0, 0.5, 1.0])
 529                 traces.append(
 530                     go.Scatter(
 531                         x=anomaly_x,
 532                         y=anomaly_y,
 533                         mode="markers",
 534                         text=hover,
 535                         hoverinfo="text+name",
 536                         showlegend=False,
 537                         legendgroup=metric,
 538                         name=metric,
 539                         marker={
 540                             "size": 15,
 541                             "symbol": "circle-open",
 542                             "color": anomaly_color,
 543                             "colorscale": C.COLORSCALE_TPUT,
 544                             "showscale": True,
 545                             "line": {
 546                                 "width": 2
 547                             },
 548                             "colorbar": {
 549                                 "y": 0.5,
 550                                 "len": 0.8,
 551                                 "title": "Circles Marking Data Classification",
 552                                 "titleside": "right",
 553                                 "tickmode": "array",
 554                                 "tickvals": [0.167, 0.500, 0.833],
 555                                 "ticktext": C.TICK_TEXT_TPUT,
 556                                 "ticks": "",
 557                                 "ticklen": 0,
 558                                 "tickangle": -90,
 559                                 "thickness": 10
 560                             }
 561                         }
 562                     )
 563                 )
 564
 565         if traces:
 566             graph = go.Figure()
 567             graph.add_traces(traces)
 568             graph.update_layout(layout.get("plot-trending-telemetry", dict()))
 569
 570         return graph
 571
 572
 573     tm_trending_graphs = list()
 574
 575     if data.empty:
 576         return tm_trending_graphs
 577
 578     for test in data.test_name.unique():
 579         df = data.loc[(data["test_name"] == test)]
 580         graph = _generate_graph(df, test, layout)
 581         if graph:
 582             tm_trending_graphs.append((graph, test, ))
 583
 584     return tm_trending_graphs