csit.infra.dash/app/cdash/trending/graphs.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Implementation of graphs for trending data.
  15 """
  16
  17 import plotly.graph_objects as go
  18 import pandas as pd
  19
  20 import hdrh.histogram
  21 import hdrh.codec
  22
  23 from ..utils.constants import Constants as C
  24 from ..utils.utils import classify_anomalies, get_color
  25
  26
  27 def _get_hdrh_latencies(row: pd.Series, name: str) -> dict:
  28     """Get the HDRH latencies from the test data.
  29
  30     :param row: A row fron the data frame with test data.
  31     :param name: The test name to be displayed as the graph title.
  32     :type row: pandas.Series
  33     :type name: str
  34     :returns: Dictionary with HDRH latencies.
  35     :rtype: dict
  36     """
  37
  38     latencies = {"name": name}
  39     for key in C.LAT_HDRH:
  40         try:
  41             latencies[key] = row[key]
  42         except KeyError:
  43             return None
  44
  45     return latencies
  46
  47
  48 def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
  49     """Select the data for graphs from the provided data frame.
  50
  51     :param data: Data frame with data for graphs.
  52     :param itm: Item (in this case job name) which data will be selected from
  53         the input data frame.
  54     :type data: pandas.DataFrame
  55     :type itm: dict
  56     :returns: A data frame with selected data.
  57     :rtype: pandas.DataFrame
  58     """
  59
  60     phy = itm["phy"].split("-")
  61     if len(phy) == 4:
  62         topo, arch, nic, drv = phy
  63         if drv == "dpdk":
  64             drv = ""
  65         else:
  66             drv += "-"
  67             drv = drv.replace("_", "-")
  68     else:
  69         return None
  70
  71     if itm["testtype"] in ("ndr", "pdr"):
  72         test_type = "ndrpdr"
  73     elif itm["testtype"] == "mrr":
  74         test_type = "mrr"
  75     elif itm["area"] == "hoststack":
  76         test_type = "hoststack"
  77     df = data.loc[(
  78         (data["test_type"] == test_type) &
  79         (data["passed"] == True)
  80     )]
  81     df = df[df.job.str.endswith(f"{topo}-{arch}")]
  82     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
  83     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
  84     df = df[df.test_id.str.contains(
  85         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
  86         regex=True
  87     )].sort_values(by="start_time", ignore_index=True)
  88
  89     return df
  90
  91
  92 def graph_trending(
  93         data: pd.DataFrame,
  94         sel: dict,
  95         layout: dict,
  96         normalize: bool
  97     ) -> tuple:
  98     """Generate the trending graph(s) - MRR, NDR, PDR and for PDR also Latences
  99     (result_latency_forward_pdr_50_avg).
 100
 101     :param data: Data frame with test results.
 102     :param sel: Selected tests.
 103     :param layout: Layout of plot.ly graph.
 104     :param normalize: If True, the data is normalized to CPU frquency
 105         Constants.NORM_FREQUENCY.
 106     :type data: pandas.DataFrame
 107     :type sel: dict
 108     :type layout: dict
 109     :type normalize: bool
 110     :returns: Trending graph(s)
 111     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
 112     """
 113
 114     if not sel:
 115         return None, None
 116
 117
 118     def _generate_trending_traces(
 119             ttype: str,
 120             name: str,
 121             df: pd.DataFrame,
 122             color: str,
 123             norm_factor: float
 124         ) -> list:
 125         """Generate the trending traces for the trending graph.
 126
 127         :param ttype: Test type (MRR, NDR, PDR).
 128         :param name: The test name to be displayed as the graph title.
 129         :param df: Data frame with test data.
 130         :param color: The color of the trace (samples and trend line).
 131         :param norm_factor: The factor used for normalization of the results to
 132             CPU frequency set to Constants.NORM_FREQUENCY.
 133         :type ttype: str
 134         :type name: str
 135         :type df: pandas.DataFrame
 136         :type color: str
 137         :type norm_factor: float
 138         :returns: Traces (samples, trending line, anomalies)
 139         :rtype: list
 140         """
 141
 142         df = df.dropna(subset=[C.VALUE[ttype], ])
 143         if df.empty:
 144             return list()
 145
 146         x_axis = df["start_time"].tolist()
 147         if ttype == "pdr-lat":
 148             y_data = [(v / norm_factor) for v in df[C.VALUE[ttype]].tolist()]
 149         else:
 150             y_data = [(v * norm_factor) for v in df[C.VALUE[ttype]].tolist()]
 151
 152         anomalies, trend_avg, trend_stdev = classify_anomalies(
 153             {k: v for k, v in zip(x_axis, y_data)}
 154         )
 155
 156         hover = list()
 157         customdata = list()
 158         customdata_samples = list()
 159         for idx, (_, row) in enumerate(df.iterrows()):
 160             d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 161             hover_itm = (
 162                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 163                 f"<prop> [{row[C.UNIT[ttype]]}]: {y_data[idx]:,.0f}<br>"
 164                 f"<stdev>"
 165                 f"{d_type}-ref: {row['dut_version']}<br>"
 166                 f"csit-ref: {row['job']}/{row['build']}<br>"
 167                 f"hosts: {', '.join(row['hosts'])}"
 168             )
 169             if ttype == "mrr":
 170                 stdev = (
 171                     f"stdev [{row['result_receive_rate_rate_unit']}]: "
 172                     f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 173                 )
 174             else:
 175                 stdev = ""
 176             hover_itm = hover_itm.replace(
 177                 "<prop>", "latency" if ttype == "pdr-lat" else "average"
 178             ).replace("<stdev>", stdev)
 179             hover.append(hover_itm)
 180             if ttype == "pdr-lat":
 181                 customdata_samples.append(_get_hdrh_latencies(row, name))
 182                 customdata.append({"name": name})
 183             else:
 184                 customdata_samples.append(
 185                     {"name": name, "show_telemetry": True}
 186                 )
 187                 customdata.append({"name": name})
 188
 189         hover_trend = list()
 190         for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 191             d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 192             hover_itm = (
 193                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 194                 f"trend [pps]: {avg:,.0f}<br>"
 195                 f"stdev [pps]: {stdev:,.0f}<br>"
 196                 f"{d_type}-ref: {row['dut_version']}<br>"
 197                 f"csit-ref: {row['job']}/{row['build']}<br>"
 198                 f"hosts: {', '.join(row['hosts'])}"
 199             )
 200             if ttype == "pdr-lat":
 201                 hover_itm = hover_itm.replace("[pps]", "[us]")
 202             hover_trend.append(hover_itm)
 203
 204         traces = [
 205             go.Scatter(  # Samples
 206                 x=x_axis,
 207                 y=y_data,
 208                 name=name,
 209                 mode="markers",
 210                 marker={
 211                     "size": 5,
 212                     "color": color,
 213                     "symbol": "circle",
 214                 },
 215                 text=hover,
 216                 hoverinfo="text+name",
 217                 showlegend=True,
 218                 legendgroup=name,
 219                 customdata=customdata_samples
 220             ),
 221             go.Scatter(  # Trend line
 222                 x=x_axis,
 223                 y=trend_avg,
 224                 name=name,
 225                 mode="lines",
 226                 line={
 227                     "shape": "linear",
 228                     "width": 1,
 229                     "color": color,
 230                 },
 231                 text=hover_trend,
 232                 hoverinfo="text+name",
 233                 showlegend=False,
 234                 legendgroup=name,
 235                 customdata=customdata
 236             )
 237         ]
 238
 239         if anomalies:
 240             anomaly_x = list()
 241             anomaly_y = list()
 242             anomaly_color = list()
 243             hover = list()
 244             for idx, anomaly in enumerate(anomalies):
 245                 if anomaly in ("regression", "progression"):
 246                     anomaly_x.append(x_axis[idx])
 247                     anomaly_y.append(trend_avg[idx])
 248                     anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 249                     hover_itm = (
 250                         f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 251                         f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
 252                         f"classification: {anomaly}"
 253                     )
 254                     if ttype == "pdr-lat":
 255                         hover_itm = hover_itm.replace("[pps]", "[us]")
 256                     hover.append(hover_itm)
 257             anomaly_color.extend([0.0, 0.5, 1.0])
 258             traces.append(
 259                 go.Scatter(
 260                     x=anomaly_x,
 261                     y=anomaly_y,
 262                     mode="markers",
 263                     text=hover,
 264                     hoverinfo="text+name",
 265                     showlegend=False,
 266                     legendgroup=name,
 267                     name=name,
 268                     customdata=customdata,
 269                     marker={
 270                         "size": 15,
 271                         "symbol": "circle-open",
 272                         "color": anomaly_color,
 273                         "colorscale": C.COLORSCALE_LAT \
 274                             if ttype == "pdr-lat" else C.COLORSCALE_TPUT,
 275                         "showscale": True,
 276                         "line": {
 277                             "width": 2
 278                         },
 279                         "colorbar": {
 280                             "y": 0.5,
 281                             "len": 0.8,
 282                             "title": "Circles Marking Data Classification",
 283                             "titleside": "right",
 284                             "tickmode": "array",
 285                             "tickvals": [0.167, 0.500, 0.833],
 286                             "ticktext": C.TICK_TEXT_LAT \
 287                                 if ttype == "pdr-lat" else C.TICK_TEXT_TPUT,
 288                             "ticks": "",
 289                             "ticklen": 0,
 290                             "tickangle": -90,
 291                             "thickness": 10
 292                         }
 293                     }
 294                 )
 295             )
 296
 297         return traces
 298
 299
 300     fig_tput = None
 301     fig_lat = None
 302     for idx, itm in enumerate(sel):
 303         df = select_trending_data(data, itm)
 304         if df is None or df.empty:
 305             continue
 306
 307         if normalize:
 308             phy = itm["phy"].split("-")
 309             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 310             norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY[topo_arch]) \
 311                 if topo_arch else 1.0
 312         else:
 313             norm_factor = 1.0
 314         traces = _generate_trending_traces(itm["testtype"], itm["id"], df,
 315             get_color(idx), norm_factor)
 316         if traces:
 317             if not fig_tput:
 318                 fig_tput = go.Figure()
 319             fig_tput.add_traces(traces)
 320
 321         if itm["testtype"] == "pdr":
 322             traces = _generate_trending_traces("pdr-lat", itm["id"], df,
 323                 get_color(idx), norm_factor)
 324             if traces:
 325                 if not fig_lat:
 326                     fig_lat = go.Figure()
 327                 fig_lat.add_traces(traces)
 328
 329     if fig_tput:
 330         fig_tput.update_layout(layout.get("plot-trending-tput", dict()))
 331     if fig_lat:
 332         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 333
 334     return fig_tput, fig_lat
 335
 336
 337 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 338     """Generate HDR Latency histogram graphs.
 339
 340     :param data: HDRH data.
 341     :param layout: Layout of plot.ly graph.
 342     :type data: dict
 343     :type layout: dict
 344     :returns: HDR latency Histogram.
 345     :rtype: plotly.graph_objects.Figure
 346     """
 347
 348     fig = None
 349
 350     traces = list()
 351     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
 352         try:
 353             decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
 354         except (hdrh.codec.HdrLengthException, TypeError):
 355             continue
 356         previous_x = 0.0
 357         prev_perc = 0.0
 358         xaxis = list()
 359         yaxis = list()
 360         hovertext = list()
 361         for item in decoded.get_recorded_iterator():
 362             # The real value is "percentile".
 363             # For 100%, we cut that down to "x_perc" to avoid
 364             # infinity.
 365             percentile = item.percentile_level_iterated_to
 366             x_perc = min(percentile, C.PERCENTILE_MAX)
 367             xaxis.append(previous_x)
 368             yaxis.append(item.value_iterated_to)
 369             hovertext.append(
 370                 f"<b>{C.GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 371                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 372                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 373                 f"Latency: {item.value_iterated_to}uSec"
 374             )
 375             next_x = 100.0 / (100.0 - x_perc)
 376             xaxis.append(next_x)
 377             yaxis.append(item.value_iterated_to)
 378             hovertext.append(
 379                 f"<b>{C.GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 380                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 381                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 382                 f"Latency: {item.value_iterated_to}uSec"
 383             )
 384             previous_x = next_x
 385             prev_perc = percentile
 386
 387         traces.append(
 388             go.Scatter(
 389                 x=xaxis,
 390                 y=yaxis,
 391                 name=C.GRAPH_LAT_HDRH_DESC[lat_name],
 392                 mode="lines",
 393                 legendgroup=C.GRAPH_LAT_HDRH_DESC[lat_name],
 394                 showlegend=bool(idx % 2),
 395                 line=dict(
 396                     color=get_color(int(idx/2)),
 397                     dash="solid",
 398                     width=1 if idx % 2 else 2
 399                 ),
 400                 hovertext=hovertext,
 401                 hoverinfo="text"
 402             )
 403         )
 404     if traces:
 405         fig = go.Figure()
 406         fig.add_traces(traces)
 407         layout_hdrh = layout.get("plot-hdrh-latency", None)
 408         if lat_hdrh:
 409             fig.update_layout(layout_hdrh)
 410
 411     return fig
 412
 413
 414 def graph_tm_trending(data: pd.DataFrame, layout: dict) -> list:
 415     """Generates one trending graph per test, each graph includes all selected
 416     metrics.
 417
 418     :param data: Data frame with telemetry data.
 419     :param layout: Layout of plot.ly graph.
 420     :type data: pandas.DataFrame
 421     :type layout: dict
 422     :returns: List of generated graphs together with test names.
 423         list(tuple(plotly.graph_objects.Figure(), str()), tuple(...), ...)
 424     :rtype: list
 425     """
 426
 427
 428     def _generate_graph(
 429             data: pd.DataFrame,
 430             test: str,
 431             layout: dict
 432         ) -> go.Figure:
 433         """Generates a trending graph for given test with all metrics.
 434
 435         :param data: Data frame with telemetry data for the given test.
 436         :param test: The name of the test.
 437         :param layout: Layout of plot.ly graph.
 438         :type data: pandas.DataFrame
 439         :type test: str
 440         :type layout: dict
 441         :returns: A trending graph.
 442         :rtype: plotly.graph_objects.Figure
 443         """
 444         graph = None
 445         traces = list()
 446         for idx, metric in enumerate(data.tm_metric.unique()):
 447             if "-pdr" in test and "='pdr'" not in metric:
 448                 continue
 449             if "-ndr" in test and "='ndr'" not in metric:
 450                 continue
 451
 452             df = data.loc[(data["tm_metric"] == metric)]
 453             x_axis = df["start_time"].tolist()
 454             y_data = [float(itm) for itm in df["tm_value"].tolist()]
 455             hover = list()
 456             for i, (_, row) in enumerate(df.iterrows()):
 457                 hover.append(
 458                     f"date: "
 459                     f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 460                     f"value: {y_data[i]:,.0f}<br>"
 461                     f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 462                     f"csit-ref: {row['job']}/{row['build']}<br>"
 463                 )
 464             if any(y_data):
 465                 anomalies, trend_avg, trend_stdev = classify_anomalies(
 466                     {k: v for k, v in zip(x_axis, y_data)}
 467                 )
 468                 hover_trend = list()
 469                 for avg, stdev, (_, row) in \
 470                         zip(trend_avg, trend_stdev, df.iterrows()):
 471                     hover_trend.append(
 472                         f"date: "
 473                         f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 474                         f"trend: {avg:,.0f}<br>"
 475                         f"stdev: {stdev:,.0f}<br>"
 476                         f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 477                         f"csit-ref: {row['job']}/{row['build']}"
 478                     )
 479             else:
 480                 anomalies = None
 481             color = get_color(idx)
 482             traces.append(
 483                 go.Scatter(  # Samples
 484                     x=x_axis,
 485                     y=y_data,
 486                     name=metric,
 487                     mode="markers",
 488                     marker={
 489                         "size": 5,
 490                         "color": color,
 491                         "symbol": "circle",
 492                     },
 493                     text=hover,
 494                     hoverinfo="text+name",
 495                     showlegend=True,
 496                     legendgroup=metric
 497                 )
 498             )
 499             if anomalies:
 500                 traces.append(
 501                     go.Scatter(  # Trend line
 502                         x=x_axis,
 503                         y=trend_avg,
 504                         name=metric,
 505                         mode="lines",
 506                         line={
 507                             "shape": "linear",
 508                             "width": 1,
 509                             "color": color,
 510                         },
 511                         text=hover_trend,
 512                         hoverinfo="text+name",
 513                         showlegend=False,
 514                         legendgroup=metric
 515                     )
 516                 )
 517
 518                 anomaly_x = list()
 519                 anomaly_y = list()
 520                 anomaly_color = list()
 521                 hover = list()
 522                 for idx, anomaly in enumerate(anomalies):
 523                     if anomaly in ("regression", "progression"):
 524                         anomaly_x.append(x_axis[idx])
 525                         anomaly_y.append(trend_avg[idx])
 526                         anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 527                         hover_itm = (
 528                             f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}"
 529                             f"<br>trend: {trend_avg[idx]:,.0f}"
 530                             f"<br>classification: {anomaly}"
 531                         )
 532                         hover.append(hover_itm)
 533                 anomaly_color.extend([0.0, 0.5, 1.0])
 534                 traces.append(
 535                     go.Scatter(
 536                         x=anomaly_x,
 537                         y=anomaly_y,
 538                         mode="markers",
 539                         text=hover,
 540                         hoverinfo="text+name",
 541                         showlegend=False,
 542                         legendgroup=metric,
 543                         name=metric,
 544                         marker={
 545                             "size": 15,
 546                             "symbol": "circle-open",
 547                             "color": anomaly_color,
 548                             "colorscale": C.COLORSCALE_TPUT,
 549                             "showscale": True,
 550                             "line": {
 551                                 "width": 2
 552                             },
 553                             "colorbar": {
 554                                 "y": 0.5,
 555                                 "len": 0.8,
 556                                 "title": "Circles Marking Data Classification",
 557                                 "titleside": "right",
 558                                 "tickmode": "array",
 559                                 "tickvals": [0.167, 0.500, 0.833],
 560                                 "ticktext": C.TICK_TEXT_TPUT,
 561                                 "ticks": "",
 562                                 "ticklen": 0,
 563                                 "tickangle": -90,
 564                                 "thickness": 10
 565                             }
 566                         }
 567                     )
 568                 )
 569
 570         if traces:
 571             graph = go.Figure()
 572             graph.add_traces(traces)
 573             graph.update_layout(layout.get("plot-trending-telemetry", dict()))
 574
 575         return graph
 576
 577
 578     tm_trending_graphs = list()
 579
 580     if data.empty:
 581         return tm_trending_graphs
 582
 583     for test in data.test_name.unique():
 584         df = data.loc[(data["test_name"] == test)]
 585         graph = _generate_graph(df, test, layout)
 586         if graph:
 587             tm_trending_graphs.append((graph, test, ))
 588
 589     return tm_trending_graphs