resources/tools/dash/app/pal/trending/graphs.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """
  15 """
  16
  17 import logging
  18 import plotly.graph_objects as go
  19 import pandas as pd
  20
  21 import hdrh.histogram
  22 import hdrh.codec
  23
  24 from datetime import datetime
  25 from numpy import isnan
  26
  27 from ..jumpavg import classify
  28
  29
  30 _NORM_FREQUENCY = 2.0  # [GHz]
  31 _FREQURENCY = {  # [GHz]
  32     "2n-aws": 1.000,
  33     "2n-dnv": 2.000,
  34     "2n-clx": 2.300,
  35     "2n-icx": 2.600,
  36     "2n-skx": 2.500,
  37     "2n-tx2": 2.500,
  38     "2n-zn2": 2.900,
  39     "3n-alt": 3.000,
  40     "3n-aws": 1.000,
  41     "3n-dnv": 2.000,
  42     "3n-icx": 2.600,
  43     "3n-skx": 2.500,
  44     "3n-tsh": 2.200
  45 }
  46
  47 _ANOMALY_COLOR = {
  48     "regression": 0.0,
  49     "normal": 0.5,
  50     "progression": 1.0
  51 }
  52 _COLORSCALE_TPUT = [
  53     [0.00, "red"],
  54     [0.33, "red"],
  55     [0.33, "white"],
  56     [0.66, "white"],
  57     [0.66, "green"],
  58     [1.00, "green"]
  59 ]
  60 _TICK_TEXT_TPUT = ["Regression", "Normal", "Progression"]
  61 _COLORSCALE_LAT = [
  62     [0.00, "green"],
  63     [0.33, "green"],
  64     [0.33, "white"],
  65     [0.66, "white"],
  66     [0.66, "red"],
  67     [1.00, "red"]
  68 ]
  69 _TICK_TEXT_LAT = ["Progression", "Normal", "Regression"]
  70 _VALUE = {
  71     "mrr": "result_receive_rate_rate_avg",
  72     "ndr": "result_ndr_lower_rate_value",
  73     "pdr": "result_pdr_lower_rate_value",
  74     "pdr-lat": "result_latency_forward_pdr_50_avg"
  75 }
  76 _UNIT = {
  77     "mrr": "result_receive_rate_rate_unit",
  78     "ndr": "result_ndr_lower_rate_unit",
  79     "pdr": "result_pdr_lower_rate_unit",
  80     "pdr-lat": "result_latency_forward_pdr_50_unit"
  81 }
  82 _LAT_HDRH = (  # Do not change the order
  83     "result_latency_forward_pdr_0_hdrh",
  84     "result_latency_reverse_pdr_0_hdrh",
  85     "result_latency_forward_pdr_10_hdrh",
  86     "result_latency_reverse_pdr_10_hdrh",
  87     "result_latency_forward_pdr_50_hdrh",
  88     "result_latency_reverse_pdr_50_hdrh",
  89     "result_latency_forward_pdr_90_hdrh",
  90     "result_latency_reverse_pdr_90_hdrh",
  91 )
  92 # This value depends on latency stream rate (9001 pps) and duration (5s).
  93 # Keep it slightly higher to ensure rounding errors to not remove tick mark.
  94 PERCENTILE_MAX = 99.999501
  95
  96 _GRAPH_LAT_HDRH_DESC = {
  97     "result_latency_forward_pdr_0_hdrh": "No-load.",
  98     "result_latency_reverse_pdr_0_hdrh": "No-load.",
  99     "result_latency_forward_pdr_10_hdrh": "Low-load, 10% PDR.",
 100     "result_latency_reverse_pdr_10_hdrh": "Low-load, 10% PDR.",
 101     "result_latency_forward_pdr_50_hdrh": "Mid-load, 50% PDR.",
 102     "result_latency_reverse_pdr_50_hdrh": "Mid-load, 50% PDR.",
 103     "result_latency_forward_pdr_90_hdrh": "High-load, 90% PDR.",
 104     "result_latency_reverse_pdr_90_hdrh": "High-load, 90% PDR."
 105 }
 106
 107
 108 def _get_color(idx: int) -> str:
 109     """
 110     """
 111     _COLORS = (
 112         "#1A1110", "#DA2647", "#214FC6", "#01786F", "#BD8260", "#FFD12A",
 113         "#A6E7FF", "#738276", "#C95A49", "#FC5A8D", "#CEC8EF", "#391285",
 114         "#6F2DA8", "#FF878D", "#45A27D", "#FFD0B9", "#FD5240", "#DB91EF",
 115         "#44D7A8", "#4F86F7", "#84DE02", "#FFCFF1", "#614051"
 116     )
 117     return _COLORS[idx % len(_COLORS)]
 118
 119
 120 def _get_hdrh_latencies(row: pd.Series, name: str) -> dict:
 121     """
 122     """
 123
 124     latencies = {"name": name}
 125     for key in _LAT_HDRH:
 126         try:
 127             latencies[key] = row[key]
 128         except KeyError:
 129             return None
 130
 131     return latencies
 132
 133
 134 def _classify_anomalies(data):
 135     """Process the data and return anomalies and trending values.
 136
 137     Gather data into groups with average as trend value.
 138     Decorate values within groups to be normal,
 139     the first value of changed average as a regression, or a progression.
 140
 141     :param data: Full data set with unavailable samples replaced by nan.
 142     :type data: OrderedDict
 143     :returns: Classification and trend values
 144     :rtype: 3-tuple, list of strings, list of floats and list of floats
 145     """
 146     # NaN means something went wrong.
 147     # Use 0.0 to cause that being reported as a severe regression.
 148     bare_data = [0.0 if isnan(sample) else sample for sample in data.values()]
 149     # TODO: Make BitCountingGroupList a subclass of list again?
 150     group_list = classify(bare_data).group_list
 151     group_list.reverse()  # Just to use .pop() for FIFO.
 152     classification = list()
 153     avgs = list()
 154     stdevs = list()
 155     active_group = None
 156     values_left = 0
 157     avg = 0.0
 158     stdv = 0.0
 159     for sample in data.values():
 160         if isnan(sample):
 161             classification.append("outlier")
 162             avgs.append(sample)
 163             stdevs.append(sample)
 164             continue
 165         if values_left < 1 or active_group is None:
 166             values_left = 0
 167             while values_left < 1:  # Ignore empty groups (should not happen).
 168                 active_group = group_list.pop()
 169                 values_left = len(active_group.run_list)
 170             avg = active_group.stats.avg
 171             stdv = active_group.stats.stdev
 172             classification.append(active_group.comment)
 173             avgs.append(avg)
 174             stdevs.append(stdv)
 175             values_left -= 1
 176             continue
 177         classification.append("normal")
 178         avgs.append(avg)
 179         stdevs.append(stdv)
 180         values_left -= 1
 181     return classification, avgs, stdevs
 182
 183
 184 def select_trending_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
 185     """
 186     """
 187
 188     phy = itm["phy"].split("-")
 189     if len(phy) == 4:
 190         topo, arch, nic, drv = phy
 191         if drv == "dpdk":
 192             drv = ""
 193         else:
 194             drv += "-"
 195             drv = drv.replace("_", "-")
 196     else:
 197         return None
 198
 199     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
 200     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
 201     dut_v100 = "none" if itm["dut"] == "trex" else itm["dut"]
 202     dut_v101 = itm["dut"]
 203
 204     df = data.loc[(
 205         (
 206             (
 207                 (data["version"] == "1.0.0") &
 208                 (data["dut_type"].str.lower() == dut_v100)
 209             ) |
 210             (
 211                 (data["version"] == "1.0.1") &
 212                 (data["dut_type"].str.lower() == dut_v101)
 213             )
 214         ) &
 215         (data["test_type"] == ttype) &
 216         (data["passed"] == True)
 217     )]
 218     df = df[df.job.str.endswith(f"{topo}-{arch}")]
 219     df = df[df.test_id.str.contains(
 220         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
 221         regex=True
 222     )].sort_values(by="start_time", ignore_index=True)
 223
 224     return df
 225
 226
 227 def _generate_trending_traces(ttype: str, name: str, df: pd.DataFrame,
 228     start: datetime, end: datetime, color: str, norm_factor: float) -> list:
 229     """
 230     """
 231
 232     df = df.dropna(subset=[_VALUE[ttype], ])
 233     if df.empty:
 234         return list()
 235     df = df.loc[((df["start_time"] >= start) & (df["start_time"] <= end))]
 236     if df.empty:
 237         return list()
 238
 239     x_axis = df["start_time"].tolist()
 240     y_data = [itm * norm_factor for itm in df[_VALUE[ttype]].tolist()]
 241
 242     anomalies, trend_avg, trend_stdev = _classify_anomalies(
 243         {k: v for k, v in zip(x_axis, y_data)}
 244     )
 245
 246     hover = list()
 247     customdata = list()
 248     for _, row in df.iterrows():
 249         d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 250         hover_itm = (
 251             f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 252             f"<prop> [{row[_UNIT[ttype]]}]: {row[_VALUE[ttype]]:,.0f}<br>"
 253             f"<stdev>"
 254             f"{d_type}-ref: {row['dut_version']}<br>"
 255             f"csit-ref: {row['job']}/{row['build']}<br>"
 256             f"hosts: {', '.join(row['hosts'])}"
 257         )
 258         if ttype == "mrr":
 259             stdev = (
 260                 f"stdev [{row['result_receive_rate_rate_unit']}]: "
 261                 f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 262             )
 263         else:
 264             stdev = ""
 265         hover_itm = hover_itm.replace(
 266             "<prop>", "latency" if ttype == "pdr-lat" else "average"
 267         ).replace("<stdev>", stdev)
 268         hover.append(hover_itm)
 269         if ttype == "pdr-lat":
 270             customdata.append(_get_hdrh_latencies(row, name))
 271
 272     hover_trend = list()
 273     for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 274         d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 275         hover_itm = (
 276             f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 277             f"trend [pps]: {avg:,.0f}<br>"
 278             f"stdev [pps]: {stdev:,.0f}<br>"
 279             f"{d_type}-ref: {row['dut_version']}<br>"
 280             f"csit-ref: {row['job']}/{row['build']}<br>"
 281             f"hosts: {', '.join(row['hosts'])}"
 282         )
 283         if ttype == "pdr-lat":
 284             hover_itm = hover_itm.replace("[pps]", "[us]")
 285         hover_trend.append(hover_itm)
 286
 287     traces = [
 288         go.Scatter(  # Samples
 289             x=x_axis,
 290             y=y_data,
 291             name=name,
 292             mode="markers",
 293             marker={
 294                 "size": 5,
 295                 "color": color,
 296                 "symbol": "circle",
 297             },
 298             text=hover,
 299             hoverinfo="text+name",
 300             showlegend=True,
 301             legendgroup=name,
 302             customdata=customdata
 303         ),
 304         go.Scatter(  # Trend line
 305             x=x_axis,
 306             y=trend_avg,
 307             name=name,
 308             mode="lines",
 309             line={
 310                 "shape": "linear",
 311                 "width": 1,
 312                 "color": color,
 313             },
 314             text=hover_trend,
 315             hoverinfo="text+name",
 316             showlegend=False,
 317             legendgroup=name,
 318         )
 319     ]
 320
 321     if anomalies:
 322         anomaly_x = list()
 323         anomaly_y = list()
 324         anomaly_color = list()
 325         hover = list()
 326         for idx, anomaly in enumerate(anomalies):
 327             if anomaly in ("regression", "progression"):
 328                 anomaly_x.append(x_axis[idx])
 329                 anomaly_y.append(trend_avg[idx])
 330                 anomaly_color.append(_ANOMALY_COLOR[anomaly])
 331                 hover_itm = (
 332                     f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 333                     f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
 334                     f"classification: {anomaly}"
 335                 )
 336                 if ttype == "pdr-lat":
 337                     hover_itm = hover_itm.replace("[pps]", "[us]")
 338                 hover.append(hover_itm)
 339         anomaly_color.extend([0.0, 0.5, 1.0])
 340         traces.append(
 341             go.Scatter(
 342                 x=anomaly_x,
 343                 y=anomaly_y,
 344                 mode="markers",
 345                 text=hover,
 346                 hoverinfo="text+name",
 347                 showlegend=False,
 348                 legendgroup=name,
 349                 name=name,
 350                 marker={
 351                     "size": 15,
 352                     "symbol": "circle-open",
 353                     "color": anomaly_color,
 354                     "colorscale": _COLORSCALE_LAT \
 355                         if ttype == "pdr-lat" else _COLORSCALE_TPUT,
 356                     "showscale": True,
 357                     "line": {
 358                         "width": 2
 359                     },
 360                     "colorbar": {
 361                         "y": 0.5,
 362                         "len": 0.8,
 363                         "title": "Circles Marking Data Classification",
 364                         "titleside": "right",
 365                         "tickmode": "array",
 366                         "tickvals": [0.167, 0.500, 0.833],
 367                         "ticktext": _TICK_TEXT_LAT \
 368                             if ttype == "pdr-lat" else _TICK_TEXT_TPUT,
 369                         "ticks": "",
 370                         "ticklen": 0,
 371                         "tickangle": -90,
 372                         "thickness": 10
 373                     }
 374                 }
 375             )
 376         )
 377
 378     return traces
 379
 380
 381 def graph_trending(data: pd.DataFrame, sel:dict, layout: dict,
 382     start: datetime, end: datetime, normalize: bool) -> tuple:
 383     """
 384     """
 385
 386     if not sel:
 387         return None, None
 388
 389     fig_tput = None
 390     fig_lat = None
 391     for idx, itm in enumerate(sel):
 392
 393         df = select_trending_data(data, itm)
 394         if df is None or df.empty:
 395             continue
 396
 397         name = "-".join((itm["dut"], itm["phy"], itm["framesize"], itm["core"],
 398             itm["test"], itm["testtype"], ))
 399         if normalize:
 400             phy = itm["phy"].split("-")
 401             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 402             norm_factor = (_NORM_FREQUENCY / _FREQURENCY[topo_arch]) \
 403                 if topo_arch else 1.0
 404         else:
 405             norm_factor = 1.0
 406         traces = _generate_trending_traces(
 407             itm["testtype"], name, df, start, end, _get_color(idx), norm_factor
 408         )
 409         if traces:
 410             if not fig_tput:
 411                 fig_tput = go.Figure()
 412             fig_tput.add_traces(traces)
 413
 414         if itm["testtype"] == "pdr":
 415             traces = _generate_trending_traces(
 416                 "pdr-lat", name, df, start, end, _get_color(idx), norm_factor
 417             )
 418             if traces:
 419                 if not fig_lat:
 420                     fig_lat = go.Figure()
 421                 fig_lat.add_traces(traces)
 422
 423     if fig_tput:
 424         fig_tput.update_layout(layout.get("plot-trending-tput", dict()))
 425     if fig_lat:
 426         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 427
 428     return fig_tput, fig_lat
 429
 430
 431 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 432     """
 433     """
 434
 435     fig = None
 436
 437     traces = list()
 438     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
 439         try:
 440             decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
 441         except (hdrh.codec.HdrLengthException, TypeError) as err:
 442             continue
 443         previous_x = 0.0
 444         prev_perc = 0.0
 445         xaxis = list()
 446         yaxis = list()
 447         hovertext = list()
 448         for item in decoded.get_recorded_iterator():
 449             # The real value is "percentile".
 450             # For 100%, we cut that down to "x_perc" to avoid
 451             # infinity.
 452             percentile = item.percentile_level_iterated_to
 453             x_perc = min(percentile, PERCENTILE_MAX)
 454             xaxis.append(previous_x)
 455             yaxis.append(item.value_iterated_to)
 456             hovertext.append(
 457                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 458                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 459                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 460                 f"Latency: {item.value_iterated_to}uSec"
 461             )
 462             next_x = 100.0 / (100.0 - x_perc)
 463             xaxis.append(next_x)
 464             yaxis.append(item.value_iterated_to)
 465             hovertext.append(
 466                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 467                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 468                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 469                 f"Latency: {item.value_iterated_to}uSec"
 470             )
 471             previous_x = next_x
 472             prev_perc = percentile
 473
 474         traces.append(
 475             go.Scatter(
 476                 x=xaxis,
 477                 y=yaxis,
 478                 name=_GRAPH_LAT_HDRH_DESC[lat_name],
 479                 mode="lines",
 480                 legendgroup=_GRAPH_LAT_HDRH_DESC[lat_name],
 481                 showlegend=bool(idx % 2),
 482                 line=dict(
 483                     color=_get_color(int(idx/2)),
 484                     dash="solid",
 485                     width=1 if idx % 2 else 2
 486                 ),
 487                 hovertext=hovertext,
 488                 hoverinfo="text"
 489             )
 490         )
 491     if traces:
 492         fig = go.Figure()
 493         fig.add_traces(traces)
 494         layout_hdrh = layout.get("plot-hdrh-latency", None)
 495         if lat_hdrh:
 496             fig.update_layout(layout_hdrh)
 497
 498     return fig