csit.infra.dash/app/cdash/trending/graphs.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Implementation of graphs for trending data.
  15 """
  16
  17 import plotly.graph_objects as go
  18 import pandas as pd
  19
  20 from ..utils.constants import Constants as C
  21 from ..utils.utils import get_color, get_hdrh_latencies
  22 from ..utils.anomalies import classify_anomalies
  23
  24
  25 def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
  26     """Select the data for graphs from the provided data frame.
  27
  28     :param data: Data frame with data for graphs.
  29     :param itm: Item (in this case job name) which data will be selected from
  30         the input data frame.
  31     :type data: pandas.DataFrame
  32     :type itm: dict
  33     :returns: A data frame with selected data.
  34     :rtype: pandas.DataFrame
  35     """
  36
  37     phy = itm["phy"].split("-")
  38     if len(phy) == 4:
  39         topo, arch, nic, drv = phy
  40         if drv == "dpdk":
  41             drv = ""
  42         else:
  43             drv += "-"
  44             drv = drv.replace("_", "-")
  45     else:
  46         return None
  47
  48     if itm["testtype"] in ("ndr", "pdr"):
  49         test_type = "ndrpdr"
  50     elif itm["testtype"] == "mrr":
  51         test_type = "mrr"
  52     elif itm["area"] == "hoststack":
  53         test_type = "hoststack"
  54     df = data.loc[(
  55         (data["test_type"] == test_type) &
  56         (data["passed"] == True)
  57     )]
  58     df = df[df.job.str.endswith(f"{topo}-{arch}")]
  59     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
  60     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
  61     df = df[df.test_id.str.contains(
  62         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
  63         regex=True
  64     )].sort_values(by="start_time", ignore_index=True)
  65
  66     return df
  67
  68
  69 def graph_trending(
  70         data: pd.DataFrame,
  71         sel: dict,
  72         layout: dict,
  73         normalize: bool
  74     ) -> tuple:
  75     """Generate the trending graph(s) - MRR, NDR, PDR and for PDR also Latences
  76     (result_latency_forward_pdr_50_avg).
  77
  78     :param data: Data frame with test results.
  79     :param sel: Selected tests.
  80     :param layout: Layout of plot.ly graph.
  81     :param normalize: If True, the data is normalized to CPU frquency
  82         Constants.NORM_FREQUENCY.
  83     :type data: pandas.DataFrame
  84     :type sel: dict
  85     :type layout: dict
  86     :type normalize: bool
  87     :returns: Trending graph(s)
  88     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
  89     """
  90
  91     if not sel:
  92         return None, None
  93
  94
  95     def _generate_trending_traces(
  96             ttype: str,
  97             name: str,
  98             df: pd.DataFrame,
  99             color: str,
 100             norm_factor: float
 101         ) -> list:
 102         """Generate the trending traces for the trending graph.
 103
 104         :param ttype: Test type (MRR, NDR, PDR).
 105         :param name: The test name to be displayed as the graph title.
 106         :param df: Data frame with test data.
 107         :param color: The color of the trace (samples and trend line).
 108         :param norm_factor: The factor used for normalization of the results to
 109             CPU frequency set to Constants.NORM_FREQUENCY.
 110         :type ttype: str
 111         :type name: str
 112         :type df: pandas.DataFrame
 113         :type color: str
 114         :type norm_factor: float
 115         :returns: Traces (samples, trending line, anomalies)
 116         :rtype: list
 117         """
 118
 119         df = df.dropna(subset=[C.VALUE[ttype], ])
 120         if df.empty:
 121             return list(), list()
 122
 123         x_axis = df["start_time"].tolist()
 124         if ttype == "latency":
 125             y_data = [(v / norm_factor) for v in df[C.VALUE[ttype]].tolist()]
 126         else:
 127             y_data = [(v * norm_factor) for v in df[C.VALUE[ttype]].tolist()]
 128         units = df[C.UNIT[ttype]].unique().tolist()
 129
 130         anomalies, trend_avg, trend_stdev = classify_anomalies(
 131             {k: v for k, v in zip(x_axis, y_data)}
 132         )
 133
 134         hover = list()
 135         customdata = list()
 136         customdata_samples = list()
 137         name_lst = name.split("-")
 138         for idx, (_, row) in enumerate(df.iterrows()):
 139             hover_itm = (
 140                 f"dut: {name_lst[0]}<br>"
 141                 f"infra: {'-'.join(name_lst[1:5])}<br>"
 142                 f"test: {'-'.join(name_lst[5:])}<br>"
 143                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 144                 f"<prop> [{row[C.UNIT[ttype]]}]: {y_data[idx]:,.0f}<br>"
 145                 f"<stdev>"
 146                 f"<additional-info>"
 147                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 148                 f"csit-ref: {row['job']}/{row['build']}<br>"
 149                 f"hosts: {', '.join(row['hosts'])}"
 150             )
 151             if ttype == "mrr":
 152                 stdev = (
 153                     f"stdev [{row['result_receive_rate_rate_unit']}]: "
 154                     f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 155                 )
 156             else:
 157                 stdev = str()
 158             if ttype in ("hoststack-cps", "hoststack-rps"):
 159                 add_info = (
 160                     f"bandwidth [{row[C.UNIT['hoststack-bps']]}]: "
 161                     f"{row[C.VALUE['hoststack-bps']]:,.0f}<br>"
 162                     f"latency [{row[C.UNIT['hoststack-latency']]}]: "
 163                     f"{row[C.VALUE['hoststack-latency']]:,.0f}<br>"
 164                 )
 165             else:
 166                 add_info = str()
 167             hover_itm = hover_itm.replace(
 168                 "<prop>", "latency" if ttype == "latency" else "average"
 169             ).replace("<stdev>", stdev).replace("<additional-info>", add_info)
 170             hover.append(hover_itm)
 171             if ttype == "latency":
 172                 customdata_samples.append(get_hdrh_latencies(row, name))
 173                 customdata.append({"name": name})
 174             else:
 175                 customdata_samples.append(
 176                     {"name": name, "show_telemetry": True}
 177                 )
 178                 customdata.append({"name": name})
 179
 180         hover_trend = list()
 181         for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 182             hover_itm = (
 183                 f"dut: {name_lst[0]}<br>"
 184                 f"infra: {'-'.join(name_lst[1:5])}<br>"
 185                 f"test: {'-'.join(name_lst[5:])}<br>"
 186                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 187                 f"trend [{row[C.UNIT[ttype]]}]: {avg:,.0f}<br>"
 188                 f"stdev [{row[C.UNIT[ttype]]}]: {stdev:,.0f}<br>"
 189                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 190                 f"csit-ref: {row['job']}/{row['build']}<br>"
 191                 f"hosts: {', '.join(row['hosts'])}"
 192             )
 193             if ttype == "latency":
 194                 hover_itm = hover_itm.replace("[pps]", "[us]")
 195             hover_trend.append(hover_itm)
 196
 197         traces = [
 198             go.Scatter(  # Samples
 199                 x=x_axis,
 200                 y=y_data,
 201                 name=name,
 202                 mode="markers",
 203                 marker={
 204                     "size": 5,
 205                     "color": color,
 206                     "symbol": "circle",
 207                 },
 208                 text=hover,
 209                 hoverinfo="text",
 210                 showlegend=True,
 211                 legendgroup=name,
 212                 customdata=customdata_samples
 213             ),
 214             go.Scatter(  # Trend line
 215                 x=x_axis,
 216                 y=trend_avg,
 217                 name=name,
 218                 mode="lines",
 219                 line={
 220                     "shape": "linear",
 221                     "width": 1,
 222                     "color": color,
 223                 },
 224                 text=hover_trend,
 225                 hoverinfo="text",
 226                 showlegend=False,
 227                 legendgroup=name,
 228                 customdata=customdata
 229             )
 230         ]
 231
 232         if anomalies:
 233             anomaly_x = list()
 234             anomaly_y = list()
 235             anomaly_color = list()
 236             hover = list()
 237             for idx, anomaly in enumerate(anomalies):
 238                 if anomaly in ("regression", "progression"):
 239                     anomaly_x.append(x_axis[idx])
 240                     anomaly_y.append(trend_avg[idx])
 241                     anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 242                     hover_itm = (
 243                         f"dut: {name_lst[0]}<br>"
 244                         f"infra: {'-'.join(name_lst[1:5])}<br>"
 245                         f"test: {'-'.join(name_lst[5:])}<br>"
 246                         f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 247                         f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
 248                         f"classification: {anomaly}"
 249                     )
 250                     if ttype == "latency":
 251                         hover_itm = hover_itm.replace("[pps]", "[us]")
 252                     hover.append(hover_itm)
 253             anomaly_color.extend([0.0, 0.5, 1.0])
 254             traces.append(
 255                 go.Scatter(
 256                     x=anomaly_x,
 257                     y=anomaly_y,
 258                     mode="markers",
 259                     text=hover,
 260                     hoverinfo="text",
 261                     showlegend=False,
 262                     legendgroup=name,
 263                     name=name,
 264                     customdata=customdata,
 265                     marker={
 266                         "size": 15,
 267                         "symbol": "circle-open",
 268                         "color": anomaly_color,
 269                         "colorscale": C.COLORSCALE_LAT \
 270                             if ttype == "latency" else C.COLORSCALE_TPUT,
 271                         "showscale": True,
 272                         "line": {
 273                             "width": 2
 274                         },
 275                         "colorbar": {
 276                             "y": 0.5,
 277                             "len": 0.8,
 278                             "title": "Circles Marking Data Classification",
 279                             "titleside": "right",
 280                             "tickmode": "array",
 281                             "tickvals": [0.167, 0.500, 0.833],
 282                             "ticktext": C.TICK_TEXT_LAT \
 283                                 if ttype == "latency" else C.TICK_TEXT_TPUT,
 284                             "ticks": "",
 285                             "ticklen": 0,
 286                             "tickangle": -90,
 287                             "thickness": 10
 288                         }
 289                     }
 290                 )
 291             )
 292
 293         return traces, units
 294
 295
 296     fig_tput = None
 297     fig_lat = None
 298     y_units = set()
 299     for idx, itm in enumerate(sel):
 300         df = select_trending_data(data, itm)
 301         if df is None or df.empty:
 302             continue
 303
 304         if normalize:
 305             phy = itm["phy"].split("-")
 306             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 307             norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY[topo_arch]) \
 308                 if topo_arch else 1.0
 309         else:
 310             norm_factor = 1.0
 311
 312         if itm["area"] == "hoststack":
 313             ttype = f"hoststack-{itm['testtype']}"
 314         else:
 315             ttype = itm["testtype"]
 316
 317         traces, units = _generate_trending_traces(
 318             ttype,
 319             itm["id"],
 320             df,
 321             get_color(idx),
 322             norm_factor
 323         )
 324         if traces:
 325             if not fig_tput:
 326                 fig_tput = go.Figure()
 327             fig_tput.add_traces(traces)
 328
 329         if itm["testtype"] == "pdr":
 330             traces, _ = _generate_trending_traces(
 331                 "latency",
 332                 itm["id"],
 333                 df,
 334                 get_color(idx),
 335                 norm_factor
 336             )
 337             if traces:
 338                 if not fig_lat:
 339                     fig_lat = go.Figure()
 340                 fig_lat.add_traces(traces)
 341
 342         y_units.update(units)
 343
 344     if fig_tput:
 345         fig_layout = layout.get("plot-trending-tput", dict())
 346         fig_layout["yaxis"]["title"] = \
 347             f"Throughput [{'|'.join(sorted(y_units))}]"
 348         fig_tput.update_layout(fig_layout)
 349     if fig_lat:
 350         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 351
 352     return fig_tput, fig_lat
 353
 354
 355 def graph_tm_trending(
 356         data: pd.DataFrame,
 357         layout: dict,
 358         all_in_one: bool=False
 359     ) -> list:
 360     """Generates one trending graph per test, each graph includes all selected
 361     metrics.
 362
 363     :param data: Data frame with telemetry data.
 364     :param layout: Layout of plot.ly graph.
 365     :param all_in_one: If True, all telemetry traces are placed in one graph,
 366         otherwise they are split to separate graphs grouped by test ID.
 367     :type data: pandas.DataFrame
 368     :type layout: dict
 369     :type all_in_one: bool
 370     :returns: List of generated graphs together with test names.
 371         list(tuple(plotly.graph_objects.Figure(), str()), tuple(...), ...)
 372     :rtype: list
 373     """
 374
 375     if data.empty:
 376         return list()
 377
 378     def _generate_traces(
 379             data: pd.DataFrame,
 380             test: str,
 381             all_in_one: bool,
 382             color_index: int
 383         ) -> list:
 384         """Generates a trending graph for given test with all metrics.
 385
 386         :param data: Data frame with telemetry data for the given test.
 387         :param test: The name of the test.
 388         :param all_in_one: If True, all telemetry traces are placed in one
 389             graph, otherwise they are split to separate graphs grouped by
 390             test ID.
 391         :param color_index: The index of the test used if all_in_one is True.
 392         :type data: pandas.DataFrame
 393         :type test: str
 394         :type all_in_one: bool
 395         :type color_index: int
 396         :returns: List of traces.
 397         :rtype: list
 398         """
 399         traces = list()
 400         metrics = data.tm_metric.unique().tolist()
 401         for idx, metric in enumerate(metrics):
 402             if "-pdr" in test and "='pdr'" not in metric:
 403                 continue
 404             if "-ndr" in test and "='ndr'" not in metric:
 405                 continue
 406
 407             df = data.loc[(data["tm_metric"] == metric)]
 408             x_axis = df["start_time"].tolist()
 409             y_data = [float(itm) for itm in df["tm_value"].tolist()]
 410             hover = list()
 411             for i, (_, row) in enumerate(df.iterrows()):
 412                 if row["test_type"] == "mrr":
 413                     rate = (
 414                         f"mrr avg [{row[C.UNIT['mrr']]}]: "
 415                         f"{row[C.VALUE['mrr']]:,.0f}<br>"
 416                         f"mrr stdev [{row[C.UNIT['mrr']]}]: "
 417                         f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 418                     )
 419                 elif row["test_type"] == "ndrpdr":
 420                     if "-pdr" in test:
 421                         rate = (
 422                             f"pdr [{row[C.UNIT['pdr']]}]: "
 423                             f"{row[C.VALUE['pdr']]:,.0f}<br>"
 424                         )
 425                     elif "-ndr" in test:
 426                         rate = (
 427                             f"ndr [{row[C.UNIT['ndr']]}]: "
 428                             f"{row[C.VALUE['ndr']]:,.0f}<br>"
 429                         )
 430                     else:
 431                         rate = str()
 432                 else:
 433                     rate = str()
 434                 hover.append(
 435                     f"date: "
 436                     f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 437                     f"value: {y_data[i]:,.2f}<br>"
 438                     f"{rate}"
 439                     f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 440                     f"csit-ref: {row['job']}/{row['build']}<br>"
 441                 )
 442             if any(y_data):
 443                 anomalies, trend_avg, trend_stdev = classify_anomalies(
 444                     {k: v for k, v in zip(x_axis, y_data)}
 445                 )
 446                 hover_trend = list()
 447                 for avg, stdev, (_, row) in \
 448                         zip(trend_avg, trend_stdev, df.iterrows()):
 449                     hover_trend.append(
 450                         f"date: "
 451                         f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 452                         f"trend: {avg:,.2f}<br>"
 453                         f"stdev: {stdev:,.2f}<br>"
 454                         f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 455                         f"csit-ref: {row['job']}/{row['build']}"
 456                     )
 457             else:
 458                 anomalies = None
 459             if all_in_one:
 460                 color = get_color(color_index * len(metrics) + idx)
 461                 metric_name = f"{test}<br>{metric}"
 462             else:
 463                 color = get_color(idx)
 464                 metric_name = metric
 465
 466             traces.append(
 467                 go.Scatter(  # Samples
 468                     x=x_axis,
 469                     y=y_data,
 470                     name=metric_name,
 471                     mode="markers",
 472                     marker={
 473                         "size": 5,
 474                         "color": color,
 475                         "symbol": "circle",
 476                     },
 477                     text=hover,
 478                     hoverinfo="text+name",
 479                     showlegend=True,
 480                     legendgroup=metric_name
 481                 )
 482             )
 483             if anomalies:
 484                 traces.append(
 485                     go.Scatter(  # Trend line
 486                         x=x_axis,
 487                         y=trend_avg,
 488                         name=metric_name,
 489                         mode="lines",
 490                         line={
 491                             "shape": "linear",
 492                             "width": 1,
 493                             "color": color,
 494                         },
 495                         text=hover_trend,
 496                         hoverinfo="text+name",
 497                         showlegend=False,
 498                         legendgroup=metric_name
 499                     )
 500                 )
 501
 502                 anomaly_x = list()
 503                 anomaly_y = list()
 504                 anomaly_color = list()
 505                 hover = list()
 506                 for idx, anomaly in enumerate(anomalies):
 507                     if anomaly in ("regression", "progression"):
 508                         anomaly_x.append(x_axis[idx])
 509                         anomaly_y.append(trend_avg[idx])
 510                         anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 511                         hover_itm = (
 512                             f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}"
 513                             f"<br>trend: {trend_avg[idx]:,.2f}"
 514                             f"<br>classification: {anomaly}"
 515                         )
 516                         hover.append(hover_itm)
 517                 anomaly_color.extend([0.0, 0.5, 1.0])
 518                 traces.append(
 519                     go.Scatter(
 520                         x=anomaly_x,
 521                         y=anomaly_y,
 522                         mode="markers",
 523                         text=hover,
 524                         hoverinfo="text+name",
 525                         showlegend=False,
 526                         legendgroup=metric_name,
 527                         name=metric_name,
 528                         marker={
 529                             "size": 15,
 530                             "symbol": "circle-open",
 531                             "color": anomaly_color,
 532                             "colorscale": C.COLORSCALE_TPUT,
 533                             "showscale": True,
 534                             "line": {
 535                                 "width": 2
 536                             },
 537                             "colorbar": {
 538                                 "y": 0.5,
 539                                 "len": 0.8,
 540                                 "title": "Circles Marking Data Classification",
 541                                 "titleside": "right",
 542                                 "tickmode": "array",
 543                                 "tickvals": [0.167, 0.500, 0.833],
 544                                 "ticktext": C.TICK_TEXT_TPUT,
 545                                 "ticks": "",
 546                                 "ticklen": 0,
 547                                 "tickangle": -90,
 548                                 "thickness": 10
 549                             }
 550                         }
 551                     )
 552                 )
 553
 554         unique_metrics = set()
 555         for itm in metrics:
 556             unique_metrics.add(itm.split("{", 1)[0])
 557         return traces, unique_metrics
 558
 559     tm_trending_graphs = list()
 560     graph_layout = layout.get("plot-trending-telemetry", dict())
 561
 562     if all_in_one:
 563         all_traces = list()
 564
 565     all_metrics = set()
 566     all_tests = list()
 567     for idx, test in enumerate(data.test_name.unique()):
 568         df = data.loc[(data["test_name"] == test)]
 569         traces, metrics = _generate_traces(df, test, all_in_one, idx)
 570         if traces:
 571             all_metrics.update(metrics)
 572             if all_in_one:
 573                 all_traces.extend(traces)
 574                 all_tests.append(test)
 575             else:
 576                 graph = go.Figure()
 577                 graph.add_traces(traces)
 578                 graph.update_layout(graph_layout)
 579                 tm_trending_graphs.append((graph, [test, ], ))
 580
 581     if all_in_one:
 582         graph = go.Figure()
 583         graph.add_traces(all_traces)
 584         graph.update_layout(graph_layout)
 585         tm_trending_graphs.append((graph, all_tests, ))
 586
 587     return tm_trending_graphs, list(all_metrics)