csit.infra.dash/app/cdash/trending/graphs.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Implementation of graphs for trending data.
  15 """
  16
  17 import plotly.graph_objects as go
  18 import pandas as pd
  19
  20 import hdrh.histogram
  21 import hdrh.codec
  22
  23 from ..utils.constants import Constants as C
  24 from ..utils.utils import classify_anomalies, get_color
  25
  26
  27 def _get_hdrh_latencies(row: pd.Series, name: str) -> dict:
  28     """Get the HDRH latencies from the test data.
  29
  30     :param row: A row fron the data frame with test data.
  31     :param name: The test name to be displayed as the graph title.
  32     :type row: pandas.Series
  33     :type name: str
  34     :returns: Dictionary with HDRH latencies.
  35     :rtype: dict
  36     """
  37
  38     latencies = {"name": name}
  39     for key in C.LAT_HDRH:
  40         try:
  41             latencies[key] = row[key]
  42         except KeyError:
  43             return None
  44
  45     return latencies
  46
  47
  48 def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
  49     """Select the data for graphs from the provided data frame.
  50
  51     :param data: Data frame with data for graphs.
  52     :param itm: Item (in this case job name) which data will be selected from
  53         the input data frame.
  54     :type data: pandas.DataFrame
  55     :type itm: dict
  56     :returns: A data frame with selected data.
  57     :rtype: pandas.DataFrame
  58     """
  59
  60     phy = itm["phy"].split("-")
  61     if len(phy) == 4:
  62         topo, arch, nic, drv = phy
  63         if drv == "dpdk":
  64             drv = ""
  65         else:
  66             drv += "-"
  67             drv = drv.replace("_", "-")
  68     else:
  69         return None
  70
  71     if itm["testtype"] in ("ndr", "pdr"):
  72         test_type = "ndrpdr"
  73     elif itm["testtype"] == "mrr":
  74         test_type = "mrr"
  75     elif itm["area"] == "hoststack":
  76         test_type = "hoststack"
  77     df = data.loc[(
  78         (data["test_type"] == test_type) &
  79         (data["passed"] == True)
  80     )]
  81     df = df[df.job.str.endswith(f"{topo}-{arch}")]
  82     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
  83     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
  84     df = df[df.test_id.str.contains(
  85         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
  86         regex=True
  87     )].sort_values(by="start_time", ignore_index=True)
  88
  89     return df
  90
  91
  92 def graph_trending(
  93         data: pd.DataFrame,
  94         sel: dict,
  95         layout: dict,
  96         normalize: bool
  97     ) -> tuple:
  98     """Generate the trending graph(s) - MRR, NDR, PDR and for PDR also Latences
  99     (result_latency_forward_pdr_50_avg).
 100
 101     :param data: Data frame with test results.
 102     :param sel: Selected tests.
 103     :param layout: Layout of plot.ly graph.
 104     :param normalize: If True, the data is normalized to CPU frquency
 105         Constants.NORM_FREQUENCY.
 106     :type data: pandas.DataFrame
 107     :type sel: dict
 108     :type layout: dict
 109     :type normalize: bool
 110     :returns: Trending graph(s)
 111     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
 112     """
 113
 114     if not sel:
 115         return None, None
 116
 117
 118     def _generate_trending_traces(
 119             ttype: str,
 120             name: str,
 121             df: pd.DataFrame,
 122             color: str,
 123             norm_factor: float
 124         ) -> list:
 125         """Generate the trending traces for the trending graph.
 126
 127         :param ttype: Test type (MRR, NDR, PDR).
 128         :param name: The test name to be displayed as the graph title.
 129         :param df: Data frame with test data.
 130         :param color: The color of the trace (samples and trend line).
 131         :param norm_factor: The factor used for normalization of the results to
 132             CPU frequency set to Constants.NORM_FREQUENCY.
 133         :type ttype: str
 134         :type name: str
 135         :type df: pandas.DataFrame
 136         :type color: str
 137         :type norm_factor: float
 138         :returns: Traces (samples, trending line, anomalies)
 139         :rtype: list
 140         """
 141
 142         df = df.dropna(subset=[C.VALUE[ttype], ])
 143         if df.empty:
 144             return list(), list()
 145
 146         x_axis = df["start_time"].tolist()
 147         if ttype == "pdr-lat":
 148             y_data = [(v / norm_factor) for v in df[C.VALUE[ttype]].tolist()]
 149         else:
 150             y_data = [(v * norm_factor) for v in df[C.VALUE[ttype]].tolist()]
 151         units = df[C.UNIT[ttype]].unique().tolist()
 152
 153         anomalies, trend_avg, trend_stdev = classify_anomalies(
 154             {k: v for k, v in zip(x_axis, y_data)}
 155         )
 156
 157         hover = list()
 158         customdata = list()
 159         customdata_samples = list()
 160         for idx, (_, row) in enumerate(df.iterrows()):
 161             d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 162             hover_itm = (
 163                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 164                 f"<prop> [{row[C.UNIT[ttype]]}]: {y_data[idx]:,.0f}<br>"
 165                 f"<stdev>"
 166                 f"<additional-info>"
 167                 f"{d_type}-ref: {row['dut_version']}<br>"
 168                 f"csit-ref: {row['job']}/{row['build']}<br>"
 169                 f"hosts: {', '.join(row['hosts'])}"
 170             )
 171             if ttype == "mrr":
 172                 stdev = (
 173                     f"stdev [{row['result_receive_rate_rate_unit']}]: "
 174                     f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 175                 )
 176             else:
 177                 stdev = str()
 178             if ttype in ("hoststack-cps", "hoststack-rps"):
 179                 add_info = (
 180                     f"bandwidth [{row[C.UNIT['hoststack-bps']]}]: "
 181                     f"{row[C.VALUE['hoststack-bps']]:,.0f}<br>"
 182                     f"latency [{row[C.UNIT['hoststack-lat']]}]: "
 183                     f"{row[C.VALUE['hoststack-lat']]:,.0f}<br>"
 184                 )
 185             else:
 186                 add_info = str()
 187             hover_itm = hover_itm.replace(
 188                 "<prop>", "latency" if ttype == "pdr-lat" else "average"
 189             ).replace("<stdev>", stdev).replace("<additional-info>", add_info)
 190             hover.append(hover_itm)
 191             if ttype == "pdr-lat":
 192                 customdata_samples.append(_get_hdrh_latencies(row, name))
 193                 customdata.append({"name": name})
 194             else:
 195                 customdata_samples.append(
 196                     {"name": name, "show_telemetry": True}
 197                 )
 198                 customdata.append({"name": name})
 199
 200         hover_trend = list()
 201         for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 202             d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 203             hover_itm = (
 204                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 205                 f"trend [{row[C.UNIT[ttype]]}]: {avg:,.0f}<br>"
 206                 f"stdev [{row[C.UNIT[ttype]]}]: {stdev:,.0f}<br>"
 207                 f"{d_type}-ref: {row['dut_version']}<br>"
 208                 f"csit-ref: {row['job']}/{row['build']}<br>"
 209                 f"hosts: {', '.join(row['hosts'])}"
 210             )
 211             if ttype == "pdr-lat":
 212                 hover_itm = hover_itm.replace("[pps]", "[us]")
 213             hover_trend.append(hover_itm)
 214
 215         traces = [
 216             go.Scatter(  # Samples
 217                 x=x_axis,
 218                 y=y_data,
 219                 name=name,
 220                 mode="markers",
 221                 marker={
 222                     "size": 5,
 223                     "color": color,
 224                     "symbol": "circle",
 225                 },
 226                 text=hover,
 227                 hoverinfo="text+name",
 228                 showlegend=True,
 229                 legendgroup=name,
 230                 customdata=customdata_samples
 231             ),
 232             go.Scatter(  # Trend line
 233                 x=x_axis,
 234                 y=trend_avg,
 235                 name=name,
 236                 mode="lines",
 237                 line={
 238                     "shape": "linear",
 239                     "width": 1,
 240                     "color": color,
 241                 },
 242                 text=hover_trend,
 243                 hoverinfo="text+name",
 244                 showlegend=False,
 245                 legendgroup=name,
 246                 customdata=customdata
 247             )
 248         ]
 249
 250         if anomalies:
 251             anomaly_x = list()
 252             anomaly_y = list()
 253             anomaly_color = list()
 254             hover = list()
 255             for idx, anomaly in enumerate(anomalies):
 256                 if anomaly in ("regression", "progression"):
 257                     anomaly_x.append(x_axis[idx])
 258                     anomaly_y.append(trend_avg[idx])
 259                     anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 260                     hover_itm = (
 261                         f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 262                         f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
 263                         f"classification: {anomaly}"
 264                     )
 265                     if ttype == "pdr-lat":
 266                         hover_itm = hover_itm.replace("[pps]", "[us]")
 267                     hover.append(hover_itm)
 268             anomaly_color.extend([0.0, 0.5, 1.0])
 269             traces.append(
 270                 go.Scatter(
 271                     x=anomaly_x,
 272                     y=anomaly_y,
 273                     mode="markers",
 274                     text=hover,
 275                     hoverinfo="text+name",
 276                     showlegend=False,
 277                     legendgroup=name,
 278                     name=name,
 279                     customdata=customdata,
 280                     marker={
 281                         "size": 15,
 282                         "symbol": "circle-open",
 283                         "color": anomaly_color,
 284                         "colorscale": C.COLORSCALE_LAT \
 285                             if ttype == "pdr-lat" else C.COLORSCALE_TPUT,
 286                         "showscale": True,
 287                         "line": {
 288                             "width": 2
 289                         },
 290                         "colorbar": {
 291                             "y": 0.5,
 292                             "len": 0.8,
 293                             "title": "Circles Marking Data Classification",
 294                             "titleside": "right",
 295                             "tickmode": "array",
 296                             "tickvals": [0.167, 0.500, 0.833],
 297                             "ticktext": C.TICK_TEXT_LAT \
 298                                 if ttype == "pdr-lat" else C.TICK_TEXT_TPUT,
 299                             "ticks": "",
 300                             "ticklen": 0,
 301                             "tickangle": -90,
 302                             "thickness": 10
 303                         }
 304                     }
 305                 )
 306             )
 307
 308         return traces, units
 309
 310
 311     fig_tput = None
 312     fig_lat = None
 313     y_units = set()
 314     for idx, itm in enumerate(sel):
 315         df = select_trending_data(data, itm)
 316         if df is None or df.empty:
 317             continue
 318
 319         if normalize:
 320             phy = itm["phy"].split("-")
 321             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 322             norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY[topo_arch]) \
 323                 if topo_arch else 1.0
 324         else:
 325             norm_factor = 1.0
 326
 327         if itm["area"] == "hoststack":
 328             ttype = f"hoststack-{itm['testtype']}"
 329         else:
 330             ttype = itm["testtype"]
 331
 332         traces, units = _generate_trending_traces(
 333             ttype,
 334             itm["id"],
 335             df,
 336             get_color(idx),
 337             norm_factor
 338         )
 339         if traces:
 340             if not fig_tput:
 341                 fig_tput = go.Figure()
 342             fig_tput.add_traces(traces)
 343
 344         if itm["testtype"] == "pdr":
 345             traces, _ = _generate_trending_traces(
 346                 "pdr-lat",
 347                 itm["id"],
 348                 df,
 349                 get_color(idx),
 350                 norm_factor
 351             )
 352             if traces:
 353                 if not fig_lat:
 354                     fig_lat = go.Figure()
 355                 fig_lat.add_traces(traces)
 356
 357         y_units.update(units)
 358
 359     if fig_tput:
 360         fig_layout = layout.get("plot-trending-tput", dict())
 361         fig_layout["yaxis"]["title"] = \
 362             f"Throughput [{'|'.join(sorted(y_units))}]"
 363         fig_tput.update_layout(fig_layout)
 364     if fig_lat:
 365         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 366
 367     return fig_tput, fig_lat
 368
 369
 370 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 371     """Generate HDR Latency histogram graphs.
 372
 373     :param data: HDRH data.
 374     :param layout: Layout of plot.ly graph.
 375     :type data: dict
 376     :type layout: dict
 377     :returns: HDR latency Histogram.
 378     :rtype: plotly.graph_objects.Figure
 379     """
 380
 381     fig = None
 382
 383     traces = list()
 384     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
 385         try:
 386             decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
 387         except (hdrh.codec.HdrLengthException, TypeError):
 388             continue
 389         previous_x = 0.0
 390         prev_perc = 0.0
 391         xaxis = list()
 392         yaxis = list()
 393         hovertext = list()
 394         for item in decoded.get_recorded_iterator():
 395             # The real value is "percentile".
 396             # For 100%, we cut that down to "x_perc" to avoid
 397             # infinity.
 398             percentile = item.percentile_level_iterated_to
 399             x_perc = min(percentile, C.PERCENTILE_MAX)
 400             xaxis.append(previous_x)
 401             yaxis.append(item.value_iterated_to)
 402             hovertext.append(
 403                 f"<b>{C.GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 404                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 405                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 406                 f"Latency: {item.value_iterated_to}uSec"
 407             )
 408             next_x = 100.0 / (100.0 - x_perc)
 409             xaxis.append(next_x)
 410             yaxis.append(item.value_iterated_to)
 411             hovertext.append(
 412                 f"<b>{C.GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 413                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 414                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 415                 f"Latency: {item.value_iterated_to}uSec"
 416             )
 417             previous_x = next_x
 418             prev_perc = percentile
 419
 420         traces.append(
 421             go.Scatter(
 422                 x=xaxis,
 423                 y=yaxis,
 424                 name=C.GRAPH_LAT_HDRH_DESC[lat_name],
 425                 mode="lines",
 426                 legendgroup=C.GRAPH_LAT_HDRH_DESC[lat_name],
 427                 showlegend=bool(idx % 2),
 428                 line=dict(
 429                     color=get_color(int(idx/2)),
 430                     dash="solid",
 431                     width=1 if idx % 2 else 2
 432                 ),
 433                 hovertext=hovertext,
 434                 hoverinfo="text"
 435             )
 436         )
 437     if traces:
 438         fig = go.Figure()
 439         fig.add_traces(traces)
 440         layout_hdrh = layout.get("plot-hdrh-latency", None)
 441         if lat_hdrh:
 442             fig.update_layout(layout_hdrh)
 443
 444     return fig
 445
 446
 447 def graph_tm_trending(data: pd.DataFrame, layout: dict) -> list:
 448     """Generates one trending graph per test, each graph includes all selected
 449     metrics.
 450
 451     :param data: Data frame with telemetry data.
 452     :param layout: Layout of plot.ly graph.
 453     :type data: pandas.DataFrame
 454     :type layout: dict
 455     :returns: List of generated graphs together with test names.
 456         list(tuple(plotly.graph_objects.Figure(), str()), tuple(...), ...)
 457     :rtype: list
 458     """
 459
 460
 461     def _generate_graph(
 462             data: pd.DataFrame,
 463             test: str,
 464             layout: dict
 465         ) -> go.Figure:
 466         """Generates a trending graph for given test with all metrics.
 467
 468         :param data: Data frame with telemetry data for the given test.
 469         :param test: The name of the test.
 470         :param layout: Layout of plot.ly graph.
 471         :type data: pandas.DataFrame
 472         :type test: str
 473         :type layout: dict
 474         :returns: A trending graph.
 475         :rtype: plotly.graph_objects.Figure
 476         """
 477         graph = None
 478         traces = list()
 479         for idx, metric in enumerate(data.tm_metric.unique()):
 480             if "-pdr" in test and "='pdr'" not in metric:
 481                 continue
 482             if "-ndr" in test and "='ndr'" not in metric:
 483                 continue
 484
 485             df = data.loc[(data["tm_metric"] == metric)]
 486             x_axis = df["start_time"].tolist()
 487             y_data = [float(itm) for itm in df["tm_value"].tolist()]
 488             hover = list()
 489             for i, (_, row) in enumerate(df.iterrows()):
 490                 if row["test_type"] == "mrr":
 491                     rate = (
 492                         f"mrr avg [{row[C.UNIT['mrr']]}]: "
 493                         f"{row[C.VALUE['mrr']]:,.0f}<br>"
 494                         f"mrr stdev [{row[C.UNIT['mrr']]}]: "
 495                         f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 496                     )
 497                 elif row["test_type"] == "ndrpdr":
 498                     if "-pdr" in test:
 499                         rate = (
 500                             f"pdr [{row[C.UNIT['pdr']]}]: "
 501                             f"{row[C.VALUE['pdr']]:,.0f}<br>"
 502                         )
 503                     elif "-ndr" in test:
 504                         rate = (
 505                             f"ndr [{row[C.UNIT['ndr']]}]: "
 506                             f"{row[C.VALUE['ndr']]:,.0f}<br>"
 507                         )
 508                     else:
 509                         rate = str()
 510                 else:
 511                     rate = str()
 512                 hover.append(
 513                     f"date: "
 514                     f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 515                     f"value: {y_data[i]:,.0f}<br>"
 516                     f"{rate}"
 517                     f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 518                     f"csit-ref: {row['job']}/{row['build']}<br>"
 519                 )
 520             if any(y_data):
 521                 anomalies, trend_avg, trend_stdev = classify_anomalies(
 522                     {k: v for k, v in zip(x_axis, y_data)}
 523                 )
 524                 hover_trend = list()
 525                 for avg, stdev, (_, row) in \
 526                         zip(trend_avg, trend_stdev, df.iterrows()):
 527                     hover_trend.append(
 528                         f"date: "
 529                         f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 530                         f"trend: {avg:,.0f}<br>"
 531                         f"stdev: {stdev:,.0f}<br>"
 532                         f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 533                         f"csit-ref: {row['job']}/{row['build']}"
 534                     )
 535             else:
 536                 anomalies = None
 537             color = get_color(idx)
 538             traces.append(
 539                 go.Scatter(  # Samples
 540                     x=x_axis,
 541                     y=y_data,
 542                     name=metric,
 543                     mode="markers",
 544                     marker={
 545                         "size": 5,
 546                         "color": color,
 547                         "symbol": "circle",
 548                     },
 549                     text=hover,
 550                     hoverinfo="text+name",
 551                     showlegend=True,
 552                     legendgroup=metric
 553                 )
 554             )
 555             if anomalies:
 556                 traces.append(
 557                     go.Scatter(  # Trend line
 558                         x=x_axis,
 559                         y=trend_avg,
 560                         name=metric,
 561                         mode="lines",
 562                         line={
 563                             "shape": "linear",
 564                             "width": 1,
 565                             "color": color,
 566                         },
 567                         text=hover_trend,
 568                         hoverinfo="text+name",
 569                         showlegend=False,
 570                         legendgroup=metric
 571                     )
 572                 )
 573
 574                 anomaly_x = list()
 575                 anomaly_y = list()
 576                 anomaly_color = list()
 577                 hover = list()
 578                 for idx, anomaly in enumerate(anomalies):
 579                     if anomaly in ("regression", "progression"):
 580                         anomaly_x.append(x_axis[idx])
 581                         anomaly_y.append(trend_avg[idx])
 582                         anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 583                         hover_itm = (
 584                             f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}"
 585                             f"<br>trend: {trend_avg[idx]:,.0f}"
 586                             f"<br>classification: {anomaly}"
 587                         )
 588                         hover.append(hover_itm)
 589                 anomaly_color.extend([0.0, 0.5, 1.0])
 590                 traces.append(
 591                     go.Scatter(
 592                         x=anomaly_x,
 593                         y=anomaly_y,
 594                         mode="markers",
 595                         text=hover,
 596                         hoverinfo="text+name",
 597                         showlegend=False,
 598                         legendgroup=metric,
 599                         name=metric,
 600                         marker={
 601                             "size": 15,
 602                             "symbol": "circle-open",
 603                             "color": anomaly_color,
 604                             "colorscale": C.COLORSCALE_TPUT,
 605                             "showscale": True,
 606                             "line": {
 607                                 "width": 2
 608                             },
 609                             "colorbar": {
 610                                 "y": 0.5,
 611                                 "len": 0.8,
 612                                 "title": "Circles Marking Data Classification",
 613                                 "titleside": "right",
 614                                 "tickmode": "array",
 615                                 "tickvals": [0.167, 0.500, 0.833],
 616                                 "ticktext": C.TICK_TEXT_TPUT,
 617                                 "ticks": "",
 618                                 "ticklen": 0,
 619                                 "tickangle": -90,
 620                                 "thickness": 10
 621                             }
 622                         }
 623                     )
 624                 )
 625
 626         if traces:
 627             graph = go.Figure()
 628             graph.add_traces(traces)
 629             graph.update_layout(layout.get("plot-trending-telemetry", dict()))
 630
 631         return graph
 632
 633
 634     tm_trending_graphs = list()
 635
 636     if data.empty:
 637         return tm_trending_graphs
 638
 639     for test in data.test_name.unique():
 640         df = data.loc[(data["test_name"] == test)]
 641         graph = _generate_graph(df, test, layout)
 642         if graph:
 643             tm_trending_graphs.append((graph, test, ))
 644
 645     return tm_trending_graphs