1 # Copyright (c) 2022 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
17 import plotly.graph_objects as go
23 from datetime import datetime
24 from numpy import isnan
26 from ..jumpavg import classify
42 _TICK_TEXT_TPUT = [u"Regression", u"Normal", u"Progression"]
51 _TICK_TEXT_LAT = [u"Progression", u"Normal", u"Regression"]
53 "mrr": "result_receive_rate_rate_avg",
54 "ndr": "result_ndr_lower_rate_value",
55 "pdr": "result_pdr_lower_rate_value",
56 "pdr-lat": "result_latency_forward_pdr_50_avg"
59 "mrr": "result_receive_rate_rate_unit",
60 "ndr": "result_ndr_lower_rate_unit",
61 "pdr": "result_pdr_lower_rate_unit",
62 "pdr-lat": "result_latency_forward_pdr_50_unit"
64 _LAT_HDRH = ( # Do not change the order
65 "result_latency_forward_pdr_0_hdrh",
66 "result_latency_reverse_pdr_0_hdrh",
67 "result_latency_forward_pdr_10_hdrh",
68 "result_latency_reverse_pdr_10_hdrh",
69 "result_latency_forward_pdr_50_hdrh",
70 "result_latency_reverse_pdr_50_hdrh",
71 "result_latency_forward_pdr_90_hdrh",
72 "result_latency_reverse_pdr_90_hdrh",
74 # This value depends on latency stream rate (9001 pps) and duration (5s).
75 # Keep it slightly higher to ensure rounding errors to not remove tick mark.
76 PERCENTILE_MAX = 99.999501
78 _GRAPH_LAT_HDRH_DESC = {
79 u"result_latency_forward_pdr_0_hdrh": u"No-load.",
80 u"result_latency_reverse_pdr_0_hdrh": u"No-load.",
81 u"result_latency_forward_pdr_10_hdrh": u"Low-load, 10% PDR.",
82 u"result_latency_reverse_pdr_10_hdrh": u"Low-load, 10% PDR.",
83 u"result_latency_forward_pdr_50_hdrh": u"Mid-load, 50% PDR.",
84 u"result_latency_reverse_pdr_50_hdrh": u"Mid-load, 50% PDR.",
85 u"result_latency_forward_pdr_90_hdrh": u"High-load, 90% PDR.",
86 u"result_latency_reverse_pdr_90_hdrh": u"High-load, 90% PDR."
90 def _get_color(idx: int) -> str:
94 "#1A1110", "#DA2647", "#214FC6", "#01786F", "#BD8260", "#FFD12A",
95 "#A6E7FF", "#738276", "#C95A49", "#FC5A8D", "#CEC8EF", "#391285",
96 "#6F2DA8", "#FF878D", "#45A27D", "#FFD0B9", "#FD5240", "#DB91EF",
97 "#44D7A8", "#4F86F7", "#84DE02", "#FFCFF1", "#614051"
99 return _COLORS[idx % len(_COLORS)]
102 def _get_hdrh_latencies(row: pd.Series, name: str) -> dict:
106 latencies = {"name": name}
107 for key in _LAT_HDRH:
109 latencies[key] = row[key]
116 def _classify_anomalies(data):
117 """Process the data and return anomalies and trending values.
119 Gather data into groups with average as trend value.
120 Decorate values within groups to be normal,
121 the first value of changed average as a regression, or a progression.
123 :param data: Full data set with unavailable samples replaced by nan.
124 :type data: OrderedDict
125 :returns: Classification and trend values
126 :rtype: 3-tuple, list of strings, list of floats and list of floats
128 # NaN means something went wrong.
129 # Use 0.0 to cause that being reported as a severe regression.
130 bare_data = [0.0 if isnan(sample) else sample for sample in data.values()]
131 # TODO: Make BitCountingGroupList a subclass of list again?
132 group_list = classify(bare_data).group_list
133 group_list.reverse() # Just to use .pop() for FIFO.
134 classification = list()
141 for sample in data.values():
143 classification.append(u"outlier")
145 stdevs.append(sample)
147 if values_left < 1 or active_group is None:
149 while values_left < 1: # Ignore empty groups (should not happen).
150 active_group = group_list.pop()
151 values_left = len(active_group.run_list)
152 avg = active_group.stats.avg
153 stdv = active_group.stats.stdev
154 classification.append(active_group.comment)
159 classification.append(u"normal")
163 return classification, avgs, stdevs
166 def select_trending_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
170 phy = itm["phy"].split("-")
172 topo, arch, nic, drv = phy
177 drv = drv.replace("_", "-")
181 core = str() if itm["dut"] == "trex" else f"{itm['core']}"
182 ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
183 dut_v100 = "none" if itm["dut"] == "trex" else itm["dut"]
184 dut_v101 = itm["dut"]
189 (data["version"] == "1.0.0") &
190 (data["dut_type"].str.lower() == dut_v100)
193 (data["version"] == "1.0.1") &
194 (data["dut_type"].str.lower() == dut_v101)
197 (data["test_type"] == ttype) &
198 (data["passed"] == True)
200 df = df[df.job.str.endswith(f"{topo}-{arch}")]
201 df = df[df.test_id.str.contains(
202 f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
204 )].sort_values(by="start_time", ignore_index=True)
209 def _generate_trending_traces(ttype: str, name: str, df: pd.DataFrame,
210 start: datetime, end: datetime, color: str) -> list:
214 df = df.dropna(subset=[_VALUE[ttype], ])
217 df = df.loc[((df["start_time"] >= start) & (df["start_time"] <= end))]
221 x_axis = df["start_time"].tolist()
223 anomalies, trend_avg, trend_stdev = _classify_anomalies(
224 {k: v for k, v in zip(x_axis, df[_VALUE[ttype]])}
229 for _, row in df.iterrows():
230 d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
232 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
233 f"<prop> [{row[_UNIT[ttype]]}]: {row[_VALUE[ttype]]:,.0f}<br>"
235 f"{d_type}-ref: {row['dut_version']}<br>"
236 f"csit-ref: {row['job']}/{row['build']}<br>"
237 f"hosts: {', '.join(row['hosts'])}"
241 f"stdev [{row['result_receive_rate_rate_unit']}]: "
242 f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
246 hover_itm = hover_itm.replace(
247 "<prop>", "latency" if ttype == "pdr-lat" else "average"
248 ).replace("<stdev>", stdev)
249 hover.append(hover_itm)
250 if ttype == "pdr-lat":
251 customdata.append(_get_hdrh_latencies(row, name))
254 for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
255 d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
257 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
258 f"trend [pps]: {avg:,.0f}<br>"
259 f"stdev [pps]: {stdev:,.0f}<br>"
260 f"{d_type}-ref: {row['dut_version']}<br>"
261 f"csit-ref: {row['job']}/{row['build']}<br>"
262 f"hosts: {', '.join(row['hosts'])}"
264 if ttype == "pdr-lat":
265 hover_itm = hover_itm.replace("[pps]", "[us]")
266 hover_trend.append(hover_itm)
269 go.Scatter( # Samples
277 u"symbol": u"circle",
280 hoverinfo=u"text+name",
283 customdata=customdata
285 go.Scatter( # Trend line
296 hoverinfo=u"text+name",
305 anomaly_color = list()
307 for idx, anomaly in enumerate(anomalies):
308 if anomaly in (u"regression", u"progression"):
309 anomaly_x.append(x_axis[idx])
310 anomaly_y.append(trend_avg[idx])
311 anomaly_color.append(_ANOMALY_COLOR[anomaly])
313 f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
314 f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
315 f"classification: {anomaly}"
317 if ttype == "pdr-lat":
318 hover_itm = hover_itm.replace("[pps]", "[us]")
319 hover.append(hover_itm)
320 anomaly_color.extend([0.0, 0.5, 1.0])
327 hoverinfo=u"text+name",
333 u"symbol": u"circle-open",
334 u"color": anomaly_color,
335 u"colorscale": _COLORSCALE_LAT \
336 if ttype == "pdr-lat" else _COLORSCALE_TPUT,
344 u"title": u"Circles Marking Data Classification",
345 u"titleside": u"right",
346 u"tickmode": u"array",
347 u"tickvals": [0.167, 0.500, 0.833],
348 u"ticktext": _TICK_TEXT_LAT \
349 if ttype == "pdr-lat" else _TICK_TEXT_TPUT,
362 def graph_trending(data: pd.DataFrame, sel:dict, layout: dict,
363 start: datetime, end: datetime) -> tuple:
372 for idx, itm in enumerate(sel):
374 df = select_trending_data(data, itm)
375 if df is None or df.empty:
378 name = "-".join((itm["dut"], itm["phy"], itm["framesize"], itm["core"],
379 itm["test"], itm["testtype"], ))
380 traces = _generate_trending_traces(
381 itm["testtype"], name, df, start, end, _get_color(idx)
385 fig_tput = go.Figure()
386 fig_tput.add_traces(traces)
388 if itm["testtype"] == "pdr":
389 traces = _generate_trending_traces(
390 "pdr-lat", name, df, start, end, _get_color(idx)
394 fig_lat = go.Figure()
395 fig_lat.add_traces(traces)
398 fig_tput.update_layout(layout.get("plot-trending-tput", dict()))
400 fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
402 return fig_tput, fig_lat
405 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
412 for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
414 decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
415 except (hdrh.codec.HdrLengthException, TypeError) as err:
422 for item in decoded.get_recorded_iterator():
423 # The real value is "percentile".
424 # For 100%, we cut that down to "x_perc" to avoid
426 percentile = item.percentile_level_iterated_to
427 x_perc = min(percentile, PERCENTILE_MAX)
428 xaxis.append(previous_x)
429 yaxis.append(item.value_iterated_to)
431 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
432 f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
433 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
434 f"Latency: {item.value_iterated_to}uSec"
436 next_x = 100.0 / (100.0 - x_perc)
438 yaxis.append(item.value_iterated_to)
440 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
441 f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
442 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
443 f"Latency: {item.value_iterated_to}uSec"
446 prev_perc = percentile
452 name=_GRAPH_LAT_HDRH_DESC[lat_name],
454 legendgroup=_GRAPH_LAT_HDRH_DESC[lat_name],
455 showlegend=bool(idx % 2),
457 color=_get_color(int(idx/2)),
459 width=1 if idx % 2 else 2
467 fig.add_traces(traces)
468 layout_hdrh = layout.get("plot-hdrh-latency", None)
470 fig.update_layout(layout_hdrh)