fcfee26e3fc5bb7c657fc1e897182bd2ff4c0231
[csit.git] / csit.infra.dash / app / cdash / trending / graphs.py
1 # Copyright (c) 2023 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Implementation of graphs for trending data.
15 """
16
17 import plotly.graph_objects as go
18 import pandas as pd
19
20 from ..utils.constants import Constants as C
21 from ..utils.utils import get_color, get_hdrh_latencies
22 from ..utils.anomalies import classify_anomalies
23
24
25 def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
26     """Select the data for graphs from the provided data frame.
27
28     :param data: Data frame with data for graphs.
29     :param itm: Item (in this case job name) which data will be selected from
30         the input data frame.
31     :type data: pandas.DataFrame
32     :type itm: dict
33     :returns: A data frame with selected data.
34     :rtype: pandas.DataFrame
35     """
36
37     phy = itm["phy"].split("-")
38     if len(phy) == 4:
39         topo, arch, nic, drv = phy
40         if drv == "dpdk":
41             drv = ""
42         else:
43             drv += "-"
44             drv = drv.replace("_", "-")
45     else:
46         return None
47
48     if itm["testtype"] in ("ndr", "pdr"):
49         test_type = "ndrpdr"
50     elif itm["testtype"] == "mrr":
51         test_type = "mrr"
52     elif itm["area"] == "hoststack":
53         test_type = "hoststack"
54     df = data.loc[(
55         (data["test_type"] == test_type) &
56         (data["passed"] == True)
57     )]
58     df = df[df.job.str.endswith(f"{topo}-{arch}")]
59     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
60     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
61     df = df[df.test_id.str.contains(
62         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
63         regex=True
64     )].sort_values(by="start_time", ignore_index=True)
65
66     return df
67
68
69 def graph_trending(
70         data: pd.DataFrame,
71         sel: dict,
72         layout: dict,
73         normalize: bool
74     ) -> tuple:
75     """Generate the trending graph(s) - MRR, NDR, PDR and for PDR also Latences
76     (result_latency_forward_pdr_50_avg).
77
78     :param data: Data frame with test results.
79     :param sel: Selected tests.
80     :param layout: Layout of plot.ly graph.
81     :param normalize: If True, the data is normalized to CPU frquency
82         Constants.NORM_FREQUENCY.
83     :type data: pandas.DataFrame
84     :type sel: dict
85     :type layout: dict
86     :type normalize: bool
87     :returns: Trending graph(s)
88     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
89     """
90
91     if not sel:
92         return None, None
93
94
95     def _generate_trending_traces(
96             ttype: str,
97             name: str,
98             df: pd.DataFrame,
99             color: str,
100             norm_factor: float
101         ) -> list:
102         """Generate the trending traces for the trending graph.
103
104         :param ttype: Test type (MRR, NDR, PDR).
105         :param name: The test name to be displayed as the graph title.
106         :param df: Data frame with test data.
107         :param color: The color of the trace (samples and trend line).
108         :param norm_factor: The factor used for normalization of the results to
109             CPU frequency set to Constants.NORM_FREQUENCY.
110         :type ttype: str
111         :type name: str
112         :type df: pandas.DataFrame
113         :type color: str
114         :type norm_factor: float
115         :returns: Traces (samples, trending line, anomalies)
116         :rtype: list
117         """
118
119         df = df.dropna(subset=[C.VALUE[ttype], ])
120         if df.empty:
121             return list(), list()
122
123         x_axis = df["start_time"].tolist()
124         if ttype == "latency":
125             y_data = [(v / norm_factor) for v in df[C.VALUE[ttype]].tolist()]
126         else:
127             y_data = [(v * norm_factor) for v in df[C.VALUE[ttype]].tolist()]
128         units = df[C.UNIT[ttype]].unique().tolist()
129
130         anomalies, trend_avg, trend_stdev = classify_anomalies(
131             {k: v for k, v in zip(x_axis, y_data)}
132         )
133
134         hover = list()
135         customdata = list()
136         customdata_samples = list()
137         name_lst = name.split("-")
138         for idx, (_, row) in enumerate(df.iterrows()):
139             hover_itm = (
140                 f"dut: {name_lst[0]}<br>"
141                 f"infra: {'-'.join(name_lst[1:5])}<br>"
142                 f"test: {'-'.join(name_lst[5:])}<br>"
143                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
144                 f"<prop> [{row[C.UNIT[ttype]]}]: {y_data[idx]:,.0f}<br>"
145                 f"<stdev>"
146                 f"<additional-info>"
147                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
148                 f"csit-ref: {row['job']}/{row['build']}<br>"
149                 f"hosts: {', '.join(row['hosts'])}"
150             )
151             if ttype == "mrr":
152                 stdev = (
153                     f"stdev [{row['result_receive_rate_rate_unit']}]: "
154                     f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
155                 )
156             else:
157                 stdev = str()
158             if ttype in ("hoststack-cps", "hoststack-rps"):
159                 add_info = (
160                     f"bandwidth [{row[C.UNIT['hoststack-bps']]}]: "
161                     f"{row[C.VALUE['hoststack-bps']]:,.0f}<br>"
162                     f"latency [{row[C.UNIT['hoststack-latency']]}]: "
163                     f"{row[C.VALUE['hoststack-latency']]:,.0f}<br>"
164                 )
165             elif ttype in ("ndr", "pdr"):  # Add mrr
166                 test_type = f"{ttype}-bandwidth"
167                 add_info = (
168                     f"bandwidth [{row[C.UNIT[test_type]]}]: "
169                     f"{row[C.VALUE[test_type]]:,.0f}<br>"
170                 )
171             else:
172                 add_info = str()
173             hover_itm = hover_itm.replace(
174                 "<prop>", "latency" if ttype == "latency" else "average"
175             ).replace("<stdev>", stdev).replace("<additional-info>", add_info)
176             hover.append(hover_itm)
177             if ttype == "latency":
178                 customdata_samples.append(get_hdrh_latencies(row, name))
179                 customdata.append({"name": name})
180             else:
181                 customdata_samples.append(
182                     {"name": name, "show_telemetry": True}
183                 )
184                 customdata.append({"name": name})
185
186         hover_trend = list()
187         for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
188             hover_itm = (
189                 f"dut: {name_lst[0]}<br>"
190                 f"infra: {'-'.join(name_lst[1:5])}<br>"
191                 f"test: {'-'.join(name_lst[5:])}<br>"
192                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
193                 f"trend [{row[C.UNIT[ttype]]}]: {avg:,.0f}<br>"
194                 f"stdev [{row[C.UNIT[ttype]]}]: {stdev:,.0f}<br>"
195                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
196                 f"csit-ref: {row['job']}/{row['build']}<br>"
197                 f"hosts: {', '.join(row['hosts'])}"
198             )
199             if ttype == "latency":
200                 hover_itm = hover_itm.replace("[pps]", "[us]")
201             hover_trend.append(hover_itm)
202
203         traces = [
204             go.Scatter(  # Samples
205                 x=x_axis,
206                 y=y_data,
207                 name=name,
208                 mode="markers",
209                 marker={
210                     "size": 5,
211                     "color": color,
212                     "symbol": "circle",
213                 },
214                 text=hover,
215                 hoverinfo="text",
216                 showlegend=True,
217                 legendgroup=name,
218                 customdata=customdata_samples
219             ),
220             go.Scatter(  # Trend line
221                 x=x_axis,
222                 y=trend_avg,
223                 name=name,
224                 mode="lines",
225                 line={
226                     "shape": "linear",
227                     "width": 1,
228                     "color": color,
229                 },
230                 text=hover_trend,
231                 hoverinfo="text",
232                 showlegend=False,
233                 legendgroup=name,
234                 customdata=customdata
235             )
236         ]
237
238         if anomalies:
239             anomaly_x = list()
240             anomaly_y = list()
241             anomaly_color = list()
242             hover = list()
243             for idx, anomaly in enumerate(anomalies):
244                 if anomaly in ("regression", "progression"):
245                     anomaly_x.append(x_axis[idx])
246                     anomaly_y.append(trend_avg[idx])
247                     anomaly_color.append(C.ANOMALY_COLOR[anomaly])
248                     hover_itm = (
249                         f"dut: {name_lst[0]}<br>"
250                         f"infra: {'-'.join(name_lst[1:5])}<br>"
251                         f"test: {'-'.join(name_lst[5:])}<br>"
252                         f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
253                         f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
254                         f"classification: {anomaly}"
255                     )
256                     if ttype == "latency":
257                         hover_itm = hover_itm.replace("[pps]", "[us]")
258                     hover.append(hover_itm)
259             anomaly_color.extend([0.0, 0.5, 1.0])
260             traces.append(
261                 go.Scatter(
262                     x=anomaly_x,
263                     y=anomaly_y,
264                     mode="markers",
265                     text=hover,
266                     hoverinfo="text",
267                     showlegend=False,
268                     legendgroup=name,
269                     name=name,
270                     customdata=customdata,
271                     marker={
272                         "size": 15,
273                         "symbol": "circle-open",
274                         "color": anomaly_color,
275                         "colorscale": C.COLORSCALE_LAT \
276                             if ttype == "latency" else C.COLORSCALE_TPUT,
277                         "showscale": True,
278                         "line": {
279                             "width": 2
280                         },
281                         "colorbar": {
282                             "y": 0.5,
283                             "len": 0.8,
284                             "title": "Circles Marking Data Classification",
285                             "titleside": "right",
286                             "tickmode": "array",
287                             "tickvals": [0.167, 0.500, 0.833],
288                             "ticktext": C.TICK_TEXT_LAT \
289                                 if ttype == "latency" else C.TICK_TEXT_TPUT,
290                             "ticks": "",
291                             "ticklen": 0,
292                             "tickangle": -90,
293                             "thickness": 10
294                         }
295                     }
296                 )
297             )
298
299         return traces, units
300
301
302     fig_tput = None
303     fig_lat = None
304     fig_band = None
305     y_units = set()
306     for idx, itm in enumerate(sel):
307         df = select_trending_data(data, itm)
308         if df is None or df.empty:
309             continue
310
311         if normalize:
312             phy = itm["phy"].split("-")
313             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
314             norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY[topo_arch]) \
315                 if topo_arch else 1.0
316         else:
317             norm_factor = 1.0
318
319         if itm["area"] == "hoststack":
320             ttype = f"hoststack-{itm['testtype']}"
321         else:
322             ttype = itm["testtype"]
323
324         traces, units = _generate_trending_traces(
325             ttype,
326             itm["id"],
327             df,
328             get_color(idx),
329             norm_factor
330         )
331         if traces:
332             if not fig_tput:
333                 fig_tput = go.Figure()
334             fig_tput.add_traces(traces)
335
336         if ttype in ("ndr", "pdr"):  # Add mrr
337             traces, _ = _generate_trending_traces(
338                 f"{ttype}-bandwidth",
339                 itm["id"],
340                 df,
341                 get_color(idx),
342                 norm_factor
343             )
344             if traces:
345                 if not fig_band:
346                     fig_band = go.Figure()
347                 fig_band.add_traces(traces)
348
349         if itm["testtype"] == "pdr":
350             traces, _ = _generate_trending_traces(
351                 "latency",
352                 itm["id"],
353                 df,
354                 get_color(idx),
355                 norm_factor
356             )
357             if traces:
358                 if not fig_lat:
359                     fig_lat = go.Figure()
360                 fig_lat.add_traces(traces)
361
362         y_units.update(units)
363
364     if fig_tput:
365         fig_layout = layout.get("plot-trending-tput", dict())
366         fig_layout["yaxis"]["title"] = \
367             f"Throughput [{'|'.join(sorted(y_units))}]"
368         fig_tput.update_layout(fig_layout)
369     if fig_band:
370         fig_band.update_layout(layout.get("plot-trending-bandwidth", dict()))
371     if fig_lat:
372         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
373
374     return fig_tput, fig_band, fig_lat
375
376
377 def graph_tm_trending(
378         data: pd.DataFrame,
379         layout: dict,
380         all_in_one: bool=False
381     ) -> list:
382     """Generates one trending graph per test, each graph includes all selected
383     metrics.
384
385     :param data: Data frame with telemetry data.
386     :param layout: Layout of plot.ly graph.
387     :param all_in_one: If True, all telemetry traces are placed in one graph,
388         otherwise they are split to separate graphs grouped by test ID.
389     :type data: pandas.DataFrame
390     :type layout: dict
391     :type all_in_one: bool
392     :returns: List of generated graphs together with test names.
393         list(tuple(plotly.graph_objects.Figure(), str()), tuple(...), ...)
394     :rtype: list
395     """
396
397     if data.empty:
398         return list()
399
400     def _generate_traces(
401             data: pd.DataFrame,
402             test: str,
403             all_in_one: bool,
404             color_index: int
405         ) -> list:
406         """Generates a trending graph for given test with all metrics.
407
408         :param data: Data frame with telemetry data for the given test.
409         :param test: The name of the test.
410         :param all_in_one: If True, all telemetry traces are placed in one
411             graph, otherwise they are split to separate graphs grouped by
412             test ID.
413         :param color_index: The index of the test used if all_in_one is True.
414         :type data: pandas.DataFrame
415         :type test: str
416         :type all_in_one: bool
417         :type color_index: int
418         :returns: List of traces.
419         :rtype: list
420         """
421         traces = list()
422         metrics = data.tm_metric.unique().tolist()
423         for idx, metric in enumerate(metrics):
424             if "-pdr" in test and "='pdr'" not in metric:
425                 continue
426             if "-ndr" in test and "='ndr'" not in metric:
427                 continue
428
429             df = data.loc[(data["tm_metric"] == metric)]
430             x_axis = df["start_time"].tolist()
431             y_data = [float(itm) for itm in df["tm_value"].tolist()]
432             hover = list()
433             for i, (_, row) in enumerate(df.iterrows()):
434                 if row["test_type"] == "mrr":
435                     rate = (
436                         f"mrr avg [{row[C.UNIT['mrr']]}]: "
437                         f"{row[C.VALUE['mrr']]:,.0f}<br>"
438                         f"mrr stdev [{row[C.UNIT['mrr']]}]: "
439                         f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
440                     )
441                 elif row["test_type"] == "ndrpdr":
442                     if "-pdr" in test:
443                         rate = (
444                             f"pdr [{row[C.UNIT['pdr']]}]: "
445                             f"{row[C.VALUE['pdr']]:,.0f}<br>"
446                         )
447                     elif "-ndr" in test:
448                         rate = (
449                             f"ndr [{row[C.UNIT['ndr']]}]: "
450                             f"{row[C.VALUE['ndr']]:,.0f}<br>"
451                         )
452                     else:
453                         rate = str()
454                 else:
455                     rate = str()
456                 hover.append(
457                     f"date: "
458                     f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
459                     f"value: {y_data[i]:,.2f}<br>"
460                     f"{rate}"
461                     f"{row['dut_type']}-ref: {row['dut_version']}<br>"
462                     f"csit-ref: {row['job']}/{row['build']}<br>"
463                 )
464             if any(y_data):
465                 anomalies, trend_avg, trend_stdev = classify_anomalies(
466                     {k: v for k, v in zip(x_axis, y_data)}
467                 )
468                 hover_trend = list()
469                 for avg, stdev, (_, row) in \
470                         zip(trend_avg, trend_stdev, df.iterrows()):
471                     hover_trend.append(
472                         f"date: "
473                         f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
474                         f"trend: {avg:,.2f}<br>"
475                         f"stdev: {stdev:,.2f}<br>"
476                         f"{row['dut_type']}-ref: {row['dut_version']}<br>"
477                         f"csit-ref: {row['job']}/{row['build']}"
478                     )
479             else:
480                 anomalies = None
481             if all_in_one:
482                 color = get_color(color_index * len(metrics) + idx)
483                 metric_name = f"{test}<br>{metric}"
484             else:
485                 color = get_color(idx)
486                 metric_name = metric
487
488             traces.append(
489                 go.Scatter(  # Samples
490                     x=x_axis,
491                     y=y_data,
492                     name=metric_name,
493                     mode="markers",
494                     marker={
495                         "size": 5,
496                         "color": color,
497                         "symbol": "circle",
498                     },
499                     text=hover,
500                     hoverinfo="text+name",
501                     showlegend=True,
502                     legendgroup=metric_name
503                 )
504             )
505             if anomalies:
506                 traces.append(
507                     go.Scatter(  # Trend line
508                         x=x_axis,
509                         y=trend_avg,
510                         name=metric_name,
511                         mode="lines",
512                         line={
513                             "shape": "linear",
514                             "width": 1,
515                             "color": color,
516                         },
517                         text=hover_trend,
518                         hoverinfo="text+name",
519                         showlegend=False,
520                         legendgroup=metric_name
521                     )
522                 )
523
524                 anomaly_x = list()
525                 anomaly_y = list()
526                 anomaly_color = list()
527                 hover = list()
528                 for idx, anomaly in enumerate(anomalies):
529                     if anomaly in ("regression", "progression"):
530                         anomaly_x.append(x_axis[idx])
531                         anomaly_y.append(trend_avg[idx])
532                         anomaly_color.append(C.ANOMALY_COLOR[anomaly])
533                         hover_itm = (
534                             f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}"
535                             f"<br>trend: {trend_avg[idx]:,.2f}"
536                             f"<br>classification: {anomaly}"
537                         )
538                         hover.append(hover_itm)
539                 anomaly_color.extend([0.0, 0.5, 1.0])
540                 traces.append(
541                     go.Scatter(
542                         x=anomaly_x,
543                         y=anomaly_y,
544                         mode="markers",
545                         text=hover,
546                         hoverinfo="text+name",
547                         showlegend=False,
548                         legendgroup=metric_name,
549                         name=metric_name,
550                         marker={
551                             "size": 15,
552                             "symbol": "circle-open",
553                             "color": anomaly_color,
554                             "colorscale": C.COLORSCALE_TPUT,
555                             "showscale": True,
556                             "line": {
557                                 "width": 2
558                             },
559                             "colorbar": {
560                                 "y": 0.5,
561                                 "len": 0.8,
562                                 "title": "Circles Marking Data Classification",
563                                 "titleside": "right",
564                                 "tickmode": "array",
565                                 "tickvals": [0.167, 0.500, 0.833],
566                                 "ticktext": C.TICK_TEXT_TPUT,
567                                 "ticks": "",
568                                 "ticklen": 0,
569                                 "tickangle": -90,
570                                 "thickness": 10
571                             }
572                         }
573                     )
574                 )
575
576         unique_metrics = set()
577         for itm in metrics:
578             unique_metrics.add(itm.split("{", 1)[0])
579         return traces, unique_metrics
580
581     tm_trending_graphs = list()
582     graph_layout = layout.get("plot-trending-telemetry", dict())
583
584     if all_in_one:
585         all_traces = list()
586
587     all_metrics = set()
588     all_tests = list()
589     for idx, test in enumerate(data.test_name.unique()):
590         df = data.loc[(data["test_name"] == test)]
591         traces, metrics = _generate_traces(df, test, all_in_one, idx)
592         if traces:
593             all_metrics.update(metrics)
594             if all_in_one:
595                 all_traces.extend(traces)
596                 all_tests.append(test)
597             else:
598                 graph = go.Figure()
599                 graph.add_traces(traces)
600                 graph.update_layout(graph_layout)
601                 tm_trending_graphs.append((graph, [test, ], ))
602
603     if all_in_one:
604         graph = go.Figure()
605         graph.add_traces(all_traces)
606         graph.update_layout(graph_layout)
607         tm_trending_graphs.append((graph, all_tests, ))
608
609     return tm_trending_graphs, list(all_metrics)