csit.infra.dash/app/cdash/trending/graphs.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Implementation of graphs for trending data.
  15 """
  16
  17 import plotly.graph_objects as go
  18 import pandas as pd
  19
  20 from ..utils.constants import Constants as C
  21 from ..utils.utils import classify_anomalies, get_color, get_hdrh_latencies
  22
  23
  24 def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
  25     """Select the data for graphs from the provided data frame.
  26
  27     :param data: Data frame with data for graphs.
  28     :param itm: Item (in this case job name) which data will be selected from
  29         the input data frame.
  30     :type data: pandas.DataFrame
  31     :type itm: dict
  32     :returns: A data frame with selected data.
  33     :rtype: pandas.DataFrame
  34     """
  35
  36     phy = itm["phy"].split("-")
  37     if len(phy) == 4:
  38         topo, arch, nic, drv = phy
  39         if drv == "dpdk":
  40             drv = ""
  41         else:
  42             drv += "-"
  43             drv = drv.replace("_", "-")
  44     else:
  45         return None
  46
  47     if itm["testtype"] in ("ndr", "pdr"):
  48         test_type = "ndrpdr"
  49     elif itm["testtype"] == "mrr":
  50         test_type = "mrr"
  51     elif itm["area"] == "hoststack":
  52         test_type = "hoststack"
  53     df = data.loc[(
  54         (data["test_type"] == test_type) &
  55         (data["passed"] == True)
  56     )]
  57     df = df[df.job.str.endswith(f"{topo}-{arch}")]
  58     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
  59     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
  60     df = df[df.test_id.str.contains(
  61         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
  62         regex=True
  63     )].sort_values(by="start_time", ignore_index=True)
  64
  65     return df
  66
  67
  68 def graph_trending(
  69         data: pd.DataFrame,
  70         sel: dict,
  71         layout: dict,
  72         normalize: bool
  73     ) -> tuple:
  74     """Generate the trending graph(s) - MRR, NDR, PDR and for PDR also Latences
  75     (result_latency_forward_pdr_50_avg).
  76
  77     :param data: Data frame with test results.
  78     :param sel: Selected tests.
  79     :param layout: Layout of plot.ly graph.
  80     :param normalize: If True, the data is normalized to CPU frquency
  81         Constants.NORM_FREQUENCY.
  82     :type data: pandas.DataFrame
  83     :type sel: dict
  84     :type layout: dict
  85     :type normalize: bool
  86     :returns: Trending graph(s)
  87     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
  88     """
  89
  90     if not sel:
  91         return None, None
  92
  93
  94     def _generate_trending_traces(
  95             ttype: str,
  96             name: str,
  97             df: pd.DataFrame,
  98             color: str,
  99             norm_factor: float
 100         ) -> list:
 101         """Generate the trending traces for the trending graph.
 102
 103         :param ttype: Test type (MRR, NDR, PDR).
 104         :param name: The test name to be displayed as the graph title.
 105         :param df: Data frame with test data.
 106         :param color: The color of the trace (samples and trend line).
 107         :param norm_factor: The factor used for normalization of the results to
 108             CPU frequency set to Constants.NORM_FREQUENCY.
 109         :type ttype: str
 110         :type name: str
 111         :type df: pandas.DataFrame
 112         :type color: str
 113         :type norm_factor: float
 114         :returns: Traces (samples, trending line, anomalies)
 115         :rtype: list
 116         """
 117
 118         df = df.dropna(subset=[C.VALUE[ttype], ])
 119         if df.empty:
 120             return list(), list()
 121
 122         x_axis = df["start_time"].tolist()
 123         if ttype == "latency":
 124             y_data = [(v / norm_factor) for v in df[C.VALUE[ttype]].tolist()]
 125         else:
 126             y_data = [(v * norm_factor) for v in df[C.VALUE[ttype]].tolist()]
 127         units = df[C.UNIT[ttype]].unique().tolist()
 128
 129         anomalies, trend_avg, trend_stdev = classify_anomalies(
 130             {k: v for k, v in zip(x_axis, y_data)}
 131         )
 132
 133         hover = list()
 134         customdata = list()
 135         customdata_samples = list()
 136         for idx, (_, row) in enumerate(df.iterrows()):
 137             d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 138             hover_itm = (
 139                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 140                 f"<prop> [{row[C.UNIT[ttype]]}]: {y_data[idx]:,.0f}<br>"
 141                 f"<stdev>"
 142                 f"<additional-info>"
 143                 f"{d_type}-ref: {row['dut_version']}<br>"
 144                 f"csit-ref: {row['job']}/{row['build']}<br>"
 145                 f"hosts: {', '.join(row['hosts'])}"
 146             )
 147             if ttype == "mrr":
 148                 stdev = (
 149                     f"stdev [{row['result_receive_rate_rate_unit']}]: "
 150                     f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 151                 )
 152             else:
 153                 stdev = str()
 154             if ttype in ("hoststack-cps", "hoststack-rps"):
 155                 add_info = (
 156                     f"bandwidth [{row[C.UNIT['hoststack-bps']]}]: "
 157                     f"{row[C.VALUE['hoststack-bps']]:,.0f}<br>"
 158                     f"latency [{row[C.UNIT['hoststack-lat']]}]: "
 159                     f"{row[C.VALUE['hoststack-lat']]:,.0f}<br>"
 160                 )
 161             else:
 162                 add_info = str()
 163             hover_itm = hover_itm.replace(
 164                 "<prop>", "latency" if ttype == "latency" else "average"
 165             ).replace("<stdev>", stdev).replace("<additional-info>", add_info)
 166             hover.append(hover_itm)
 167             if ttype == "latency":
 168                 customdata_samples.append(get_hdrh_latencies(row, name))
 169                 customdata.append({"name": name})
 170             else:
 171                 customdata_samples.append(
 172                     {"name": name, "show_telemetry": True}
 173                 )
 174                 customdata.append({"name": name})
 175
 176         hover_trend = list()
 177         for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 178             d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 179             hover_itm = (
 180                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 181                 f"trend [{row[C.UNIT[ttype]]}]: {avg:,.0f}<br>"
 182                 f"stdev [{row[C.UNIT[ttype]]}]: {stdev:,.0f}<br>"
 183                 f"{d_type}-ref: {row['dut_version']}<br>"
 184                 f"csit-ref: {row['job']}/{row['build']}<br>"
 185                 f"hosts: {', '.join(row['hosts'])}"
 186             )
 187             if ttype == "latency":
 188                 hover_itm = hover_itm.replace("[pps]", "[us]")
 189             hover_trend.append(hover_itm)
 190
 191         traces = [
 192             go.Scatter(  # Samples
 193                 x=x_axis,
 194                 y=y_data,
 195                 name=name,
 196                 mode="markers",
 197                 marker={
 198                     "size": 5,
 199                     "color": color,
 200                     "symbol": "circle",
 201                 },
 202                 text=hover,
 203                 hoverinfo="text+name",
 204                 showlegend=True,
 205                 legendgroup=name,
 206                 customdata=customdata_samples
 207             ),
 208             go.Scatter(  # Trend line
 209                 x=x_axis,
 210                 y=trend_avg,
 211                 name=name,
 212                 mode="lines",
 213                 line={
 214                     "shape": "linear",
 215                     "width": 1,
 216                     "color": color,
 217                 },
 218                 text=hover_trend,
 219                 hoverinfo="text+name",
 220                 showlegend=False,
 221                 legendgroup=name,
 222                 customdata=customdata
 223             )
 224         ]
 225
 226         if anomalies:
 227             anomaly_x = list()
 228             anomaly_y = list()
 229             anomaly_color = list()
 230             hover = list()
 231             for idx, anomaly in enumerate(anomalies):
 232                 if anomaly in ("regression", "progression"):
 233                     anomaly_x.append(x_axis[idx])
 234                     anomaly_y.append(trend_avg[idx])
 235                     anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 236                     hover_itm = (
 237                         f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 238                         f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
 239                         f"classification: {anomaly}"
 240                     )
 241                     if ttype == "latency":
 242                         hover_itm = hover_itm.replace("[pps]", "[us]")
 243                     hover.append(hover_itm)
 244             anomaly_color.extend([0.0, 0.5, 1.0])
 245             traces.append(
 246                 go.Scatter(
 247                     x=anomaly_x,
 248                     y=anomaly_y,
 249                     mode="markers",
 250                     text=hover,
 251                     hoverinfo="text+name",
 252                     showlegend=False,
 253                     legendgroup=name,
 254                     name=name,
 255                     customdata=customdata,
 256                     marker={
 257                         "size": 15,
 258                         "symbol": "circle-open",
 259                         "color": anomaly_color,
 260                         "colorscale": C.COLORSCALE_LAT \
 261                             if ttype == "latency" else C.COLORSCALE_TPUT,
 262                         "showscale": True,
 263                         "line": {
 264                             "width": 2
 265                         },
 266                         "colorbar": {
 267                             "y": 0.5,
 268                             "len": 0.8,
 269                             "title": "Circles Marking Data Classification",
 270                             "titleside": "right",
 271                             "tickmode": "array",
 272                             "tickvals": [0.167, 0.500, 0.833],
 273                             "ticktext": C.TICK_TEXT_LAT \
 274                                 if ttype == "latency" else C.TICK_TEXT_TPUT,
 275                             "ticks": "",
 276                             "ticklen": 0,
 277                             "tickangle": -90,
 278                             "thickness": 10
 279                         }
 280                     }
 281                 )
 282             )
 283
 284         return traces, units
 285
 286
 287     fig_tput = None
 288     fig_lat = None
 289     y_units = set()
 290     for idx, itm in enumerate(sel):
 291         df = select_trending_data(data, itm)
 292         if df is None or df.empty:
 293             continue
 294
 295         if normalize:
 296             phy = itm["phy"].split("-")
 297             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 298             norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY[topo_arch]) \
 299                 if topo_arch else 1.0
 300         else:
 301             norm_factor = 1.0
 302
 303         if itm["area"] == "hoststack":
 304             ttype = f"hoststack-{itm['testtype']}"
 305         else:
 306             ttype = itm["testtype"]
 307
 308         traces, units = _generate_trending_traces(
 309             ttype,
 310             itm["id"],
 311             df,
 312             get_color(idx),
 313             norm_factor
 314         )
 315         if traces:
 316             if not fig_tput:
 317                 fig_tput = go.Figure()
 318             fig_tput.add_traces(traces)
 319
 320         if itm["testtype"] == "pdr":
 321             traces, _ = _generate_trending_traces(
 322                 "latency",
 323                 itm["id"],
 324                 df,
 325                 get_color(idx),
 326                 norm_factor
 327             )
 328             if traces:
 329                 if not fig_lat:
 330                     fig_lat = go.Figure()
 331                 fig_lat.add_traces(traces)
 332
 333         y_units.update(units)
 334
 335     if fig_tput:
 336         fig_layout = layout.get("plot-trending-tput", dict())
 337         fig_layout["yaxis"]["title"] = \
 338             f"Throughput [{'|'.join(sorted(y_units))}]"
 339         fig_tput.update_layout(fig_layout)
 340     if fig_lat:
 341         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 342
 343     return fig_tput, fig_lat
 344
 345
 346 def graph_tm_trending(data: pd.DataFrame, layout: dict) -> list:
 347     """Generates one trending graph per test, each graph includes all selected
 348     metrics.
 349
 350     :param data: Data frame with telemetry data.
 351     :param layout: Layout of plot.ly graph.
 352     :type data: pandas.DataFrame
 353     :type layout: dict
 354     :returns: List of generated graphs together with test names.
 355         list(tuple(plotly.graph_objects.Figure(), str()), tuple(...), ...)
 356     :rtype: list
 357     """
 358
 359
 360     def _generate_graph(
 361             data: pd.DataFrame,
 362             test: str,
 363             layout: dict
 364         ) -> go.Figure:
 365         """Generates a trending graph for given test with all metrics.
 366
 367         :param data: Data frame with telemetry data for the given test.
 368         :param test: The name of the test.
 369         :param layout: Layout of plot.ly graph.
 370         :type data: pandas.DataFrame
 371         :type test: str
 372         :type layout: dict
 373         :returns: A trending graph.
 374         :rtype: plotly.graph_objects.Figure
 375         """
 376         graph = None
 377         traces = list()
 378         for idx, metric in enumerate(data.tm_metric.unique()):
 379             if "-pdr" in test and "='pdr'" not in metric:
 380                 continue
 381             if "-ndr" in test and "='ndr'" not in metric:
 382                 continue
 383
 384             df = data.loc[(data["tm_metric"] == metric)]
 385             x_axis = df["start_time"].tolist()
 386             y_data = [float(itm) for itm in df["tm_value"].tolist()]
 387             hover = list()
 388             for i, (_, row) in enumerate(df.iterrows()):
 389                 if row["test_type"] == "mrr":
 390                     rate = (
 391                         f"mrr avg [{row[C.UNIT['mrr']]}]: "
 392                         f"{row[C.VALUE['mrr']]:,.0f}<br>"
 393                         f"mrr stdev [{row[C.UNIT['mrr']]}]: "
 394                         f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 395                     )
 396                 elif row["test_type"] == "ndrpdr":
 397                     if "-pdr" in test:
 398                         rate = (
 399                             f"pdr [{row[C.UNIT['pdr']]}]: "
 400                             f"{row[C.VALUE['pdr']]:,.0f}<br>"
 401                         )
 402                     elif "-ndr" in test:
 403                         rate = (
 404                             f"ndr [{row[C.UNIT['ndr']]}]: "
 405                             f"{row[C.VALUE['ndr']]:,.0f}<br>"
 406                         )
 407                     else:
 408                         rate = str()
 409                 else:
 410                     rate = str()
 411                 hover.append(
 412                     f"date: "
 413                     f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 414                     f"value: {y_data[i]:,.0f}<br>"
 415                     f"{rate}"
 416                     f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 417                     f"csit-ref: {row['job']}/{row['build']}<br>"
 418                 )
 419             if any(y_data):
 420                 anomalies, trend_avg, trend_stdev = classify_anomalies(
 421                     {k: v for k, v in zip(x_axis, y_data)}
 422                 )
 423                 hover_trend = list()
 424                 for avg, stdev, (_, row) in \
 425                         zip(trend_avg, trend_stdev, df.iterrows()):
 426                     hover_trend.append(
 427                         f"date: "
 428                         f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 429                         f"trend: {avg:,.0f}<br>"
 430                         f"stdev: {stdev:,.0f}<br>"
 431                         f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 432                         f"csit-ref: {row['job']}/{row['build']}"
 433                     )
 434             else:
 435                 anomalies = None
 436             color = get_color(idx)
 437             traces.append(
 438                 go.Scatter(  # Samples
 439                     x=x_axis,
 440                     y=y_data,
 441                     name=metric,
 442                     mode="markers",
 443                     marker={
 444                         "size": 5,
 445                         "color": color,
 446                         "symbol": "circle",
 447                     },
 448                     text=hover,
 449                     hoverinfo="text+name",
 450                     showlegend=True,
 451                     legendgroup=metric
 452                 )
 453             )
 454             if anomalies:
 455                 traces.append(
 456                     go.Scatter(  # Trend line
 457                         x=x_axis,
 458                         y=trend_avg,
 459                         name=metric,
 460                         mode="lines",
 461                         line={
 462                             "shape": "linear",
 463                             "width": 1,
 464                             "color": color,
 465                         },
 466                         text=hover_trend,
 467                         hoverinfo="text+name",
 468                         showlegend=False,
 469                         legendgroup=metric
 470                     )
 471                 )
 472
 473                 anomaly_x = list()
 474                 anomaly_y = list()
 475                 anomaly_color = list()
 476                 hover = list()
 477                 for idx, anomaly in enumerate(anomalies):
 478                     if anomaly in ("regression", "progression"):
 479                         anomaly_x.append(x_axis[idx])
 480                         anomaly_y.append(trend_avg[idx])
 481                         anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 482                         hover_itm = (
 483                             f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}"
 484                             f"<br>trend: {trend_avg[idx]:,.0f}"
 485                             f"<br>classification: {anomaly}"
 486                         )
 487                         hover.append(hover_itm)
 488                 anomaly_color.extend([0.0, 0.5, 1.0])
 489                 traces.append(
 490                     go.Scatter(
 491                         x=anomaly_x,
 492                         y=anomaly_y,
 493                         mode="markers",
 494                         text=hover,
 495                         hoverinfo="text+name",
 496                         showlegend=False,
 497                         legendgroup=metric,
 498                         name=metric,
 499                         marker={
 500                             "size": 15,
 501                             "symbol": "circle-open",
 502                             "color": anomaly_color,
 503                             "colorscale": C.COLORSCALE_TPUT,
 504                             "showscale": True,
 505                             "line": {
 506                                 "width": 2
 507                             },
 508                             "colorbar": {
 509                                 "y": 0.5,
 510                                 "len": 0.8,
 511                                 "title": "Circles Marking Data Classification",
 512                                 "titleside": "right",
 513                                 "tickmode": "array",
 514                                 "tickvals": [0.167, 0.500, 0.833],
 515                                 "ticktext": C.TICK_TEXT_TPUT,
 516                                 "ticks": "",
 517                                 "ticklen": 0,
 518                                 "tickangle": -90,
 519                                 "thickness": 10
 520                             }
 521                         }
 522                     )
 523                 )
 524
 525         if traces:
 526             graph = go.Figure()
 527             graph.add_traces(traces)
 528             graph.update_layout(layout.get("plot-trending-telemetry", dict()))
 529
 530         return graph
 531
 532
 533     tm_trending_graphs = list()
 534
 535     if data.empty:
 536         return tm_trending_graphs
 537
 538     for test in data.test_name.unique():
 539         df = data.loc[(data["test_name"] == test)]
 540         graph = _generate_graph(df, test, layout)
 541         if graph:
 542             tm_trending_graphs.append((graph, test, ))
 543
 544     return tm_trending_graphs