resources/tools/dash/app/pal/trending/graphs.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """
  15 """
  16
  17 import logging
  18 import plotly.graph_objects as go
  19 import pandas as pd
  20
  21 import hdrh.histogram
  22 import hdrh.codec
  23
  24 from datetime import datetime
  25 from numpy import isnan
  26
  27 from ..jumpavg import classify
  28
  29
  30 _NORM_FREQUENCY = 2.0  # [GHz]
  31 _FREQURENCY = {  # [GHz]
  32     "2n-aws": 1.000,
  33     "2n-dnv": 2.000,
  34     "2n-clx": 2.300,
  35     "2n-icx": 2.600,
  36     "2n-skx": 2.500,
  37     "2n-tx2": 2.500,
  38     "2n-zn2": 2.900,
  39     "3n-alt": 3.000,
  40     "3n-aws": 1.000,
  41     "3n-dnv": 2.000,
  42     "3n-icx": 2.600,
  43     "3n-skx": 2.500,
  44     "3n-tsh": 2.200
  45 }
  46
  47 _ANOMALY_COLOR = {
  48     "regression": 0.0,
  49     "normal": 0.5,
  50     "progression": 1.0
  51 }
  52 _COLORSCALE_TPUT = [
  53     [0.00, "red"],
  54     [0.33, "red"],
  55     [0.33, "white"],
  56     [0.66, "white"],
  57     [0.66, "green"],
  58     [1.00, "green"]
  59 ]
  60 _TICK_TEXT_TPUT = ["Regression", "Normal", "Progression"]
  61 _COLORSCALE_LAT = [
  62     [0.00, "green"],
  63     [0.33, "green"],
  64     [0.33, "white"],
  65     [0.66, "white"],
  66     [0.66, "red"],
  67     [1.00, "red"]
  68 ]
  69 _TICK_TEXT_LAT = ["Progression", "Normal", "Regression"]
  70 _VALUE = {
  71     "mrr": "result_receive_rate_rate_avg",
  72     "ndr": "result_ndr_lower_rate_value",
  73     "pdr": "result_pdr_lower_rate_value",
  74     "pdr-lat": "result_latency_forward_pdr_50_avg"
  75 }
  76 _UNIT = {
  77     "mrr": "result_receive_rate_rate_unit",
  78     "ndr": "result_ndr_lower_rate_unit",
  79     "pdr": "result_pdr_lower_rate_unit",
  80     "pdr-lat": "result_latency_forward_pdr_50_unit"
  81 }
  82 _LAT_HDRH = (  # Do not change the order
  83     "result_latency_forward_pdr_0_hdrh",
  84     "result_latency_reverse_pdr_0_hdrh",
  85     "result_latency_forward_pdr_10_hdrh",
  86     "result_latency_reverse_pdr_10_hdrh",
  87     "result_latency_forward_pdr_50_hdrh",
  88     "result_latency_reverse_pdr_50_hdrh",
  89     "result_latency_forward_pdr_90_hdrh",
  90     "result_latency_reverse_pdr_90_hdrh",
  91 )
  92 # This value depends on latency stream rate (9001 pps) and duration (5s).
  93 # Keep it slightly higher to ensure rounding errors to not remove tick mark.
  94 PERCENTILE_MAX = 99.999501
  95
  96 _GRAPH_LAT_HDRH_DESC = {
  97     "result_latency_forward_pdr_0_hdrh": "No-load.",
  98     "result_latency_reverse_pdr_0_hdrh": "No-load.",
  99     "result_latency_forward_pdr_10_hdrh": "Low-load, 10% PDR.",
 100     "result_latency_reverse_pdr_10_hdrh": "Low-load, 10% PDR.",
 101     "result_latency_forward_pdr_50_hdrh": "Mid-load, 50% PDR.",
 102     "result_latency_reverse_pdr_50_hdrh": "Mid-load, 50% PDR.",
 103     "result_latency_forward_pdr_90_hdrh": "High-load, 90% PDR.",
 104     "result_latency_reverse_pdr_90_hdrh": "High-load, 90% PDR."
 105 }
 106
 107
 108 def _get_color(idx: int) -> str:
 109     """
 110     """
 111     _COLORS = (
 112         "#1A1110", "#DA2647", "#214FC6", "#01786F", "#BD8260", "#FFD12A",
 113         "#A6E7FF", "#738276", "#C95A49", "#FC5A8D", "#CEC8EF", "#391285",
 114         "#6F2DA8", "#FF878D", "#45A27D", "#FFD0B9", "#FD5240", "#DB91EF",
 115         "#44D7A8", "#4F86F7", "#84DE02", "#FFCFF1", "#614051"
 116     )
 117     return _COLORS[idx % len(_COLORS)]
 118
 119
 120 def _get_hdrh_latencies(row: pd.Series, name: str) -> dict:
 121     """
 122     """
 123
 124     latencies = {"name": name}
 125     for key in _LAT_HDRH:
 126         try:
 127             latencies[key] = row[key]
 128         except KeyError:
 129             return None
 130
 131     return latencies
 132
 133
 134 def _classify_anomalies(data):
 135     """Process the data and return anomalies and trending values.
 136
 137     Gather data into groups with average as trend value.
 138     Decorate values within groups to be normal,
 139     the first value of changed average as a regression, or a progression.
 140
 141     :param data: Full data set with unavailable samples replaced by nan.
 142     :type data: OrderedDict
 143     :returns: Classification and trend values
 144     :rtype: 3-tuple, list of strings, list of floats and list of floats
 145     """
 146     # NaN means something went wrong.
 147     # Use 0.0 to cause that being reported as a severe regression.
 148     bare_data = [0.0 if isnan(sample) else sample for sample in data.values()]
 149     # TODO: Make BitCountingGroupList a subclass of list again?
 150     group_list = classify(bare_data).group_list
 151     group_list.reverse()  # Just to use .pop() for FIFO.
 152     classification = list()
 153     avgs = list()
 154     stdevs = list()
 155     active_group = None
 156     values_left = 0
 157     avg = 0.0
 158     stdv = 0.0
 159     for sample in data.values():
 160         if isnan(sample):
 161             classification.append("outlier")
 162             avgs.append(sample)
 163             stdevs.append(sample)
 164             continue
 165         if values_left < 1 or active_group is None:
 166             values_left = 0
 167             while values_left < 1:  # Ignore empty groups (should not happen).
 168                 active_group = group_list.pop()
 169                 values_left = len(active_group.run_list)
 170             avg = active_group.stats.avg
 171             stdv = active_group.stats.stdev
 172             classification.append(active_group.comment)
 173             avgs.append(avg)
 174             stdevs.append(stdv)
 175             values_left -= 1
 176             continue
 177         classification.append("normal")
 178         avgs.append(avg)
 179         stdevs.append(stdv)
 180         values_left -= 1
 181     return classification, avgs, stdevs
 182
 183
 184 def select_trending_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
 185     """
 186     """
 187
 188     phy = itm["phy"].split("-")
 189     if len(phy) == 4:
 190         topo, arch, nic, drv = phy
 191         if drv == "dpdk":
 192             drv = ""
 193         else:
 194             drv += "-"
 195             drv = drv.replace("_", "-")
 196     else:
 197         return None
 198
 199     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
 200     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
 201     dut_v100 = "none" if itm["dut"] == "trex" else itm["dut"]
 202     dut_v101 = itm["dut"]
 203
 204     df = data.loc[(
 205         (
 206             (
 207                 (data["version"] == "1.0.0") &
 208                 (data["dut_type"].str.lower() == dut_v100)
 209             ) |
 210             (
 211                 (data["version"] == "1.0.1") &
 212                 (data["dut_type"].str.lower() == dut_v101)
 213             )
 214         ) &
 215         (data["test_type"] == ttype) &
 216         (data["passed"] == True)
 217     )]
 218     df = df[df.job.str.endswith(f"{topo}-{arch}")]
 219     df = df[df.test_id.str.contains(
 220         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
 221         regex=True
 222     )].sort_values(by="start_time", ignore_index=True)
 223
 224     return df
 225
 226
 227 def _generate_trending_traces(ttype: str, name: str, df: pd.DataFrame,
 228     start: datetime, end: datetime, color: str, norm_factor: float) -> list:
 229     """
 230     """
 231
 232     df = df.dropna(subset=[_VALUE[ttype], ])
 233     if df.empty:
 234         return list()
 235     df = df.loc[((df["start_time"] >= start) & (df["start_time"] <= end))]
 236     if df.empty:
 237         return list()
 238
 239     x_axis = df["start_time"].tolist()
 240     if ttype == "pdr-lat":
 241         y_data = [(itm / norm_factor) for itm in df[_VALUE[ttype]].tolist()]
 242     else:
 243         y_data = [(itm * norm_factor) for itm in df[_VALUE[ttype]].tolist()]
 244
 245     anomalies, trend_avg, trend_stdev = _classify_anomalies(
 246         {k: v for k, v in zip(x_axis, y_data)}
 247     )
 248
 249     hover = list()
 250     customdata = list()
 251     for idx, (_, row) in enumerate(df.iterrows()):
 252         d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 253         hover_itm = (
 254             f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 255             f"<prop> [{row[_UNIT[ttype]]}]: {y_data[idx]:,.0f}<br>"
 256             f"<stdev>"
 257             f"{d_type}-ref: {row['dut_version']}<br>"
 258             f"csit-ref: {row['job']}/{row['build']}<br>"
 259             f"hosts: {', '.join(row['hosts'])}"
 260         )
 261         if ttype == "mrr":
 262             stdev = (
 263                 f"stdev [{row['result_receive_rate_rate_unit']}]: "
 264                 f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 265             )
 266         else:
 267             stdev = ""
 268         hover_itm = hover_itm.replace(
 269             "<prop>", "latency" if ttype == "pdr-lat" else "average"
 270         ).replace("<stdev>", stdev)
 271         hover.append(hover_itm)
 272         if ttype == "pdr-lat":
 273             customdata.append(_get_hdrh_latencies(row, name))
 274
 275     hover_trend = list()
 276     for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 277         d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 278         hover_itm = (
 279             f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 280             f"trend [pps]: {avg:,.0f}<br>"
 281             f"stdev [pps]: {stdev:,.0f}<br>"
 282             f"{d_type}-ref: {row['dut_version']}<br>"
 283             f"csit-ref: {row['job']}/{row['build']}<br>"
 284             f"hosts: {', '.join(row['hosts'])}"
 285         )
 286         if ttype == "pdr-lat":
 287             hover_itm = hover_itm.replace("[pps]", "[us]")
 288         hover_trend.append(hover_itm)
 289
 290     traces = [
 291         go.Scatter(  # Samples
 292             x=x_axis,
 293             y=y_data,
 294             name=name,
 295             mode="markers",
 296             marker={
 297                 "size": 5,
 298                 "color": color,
 299                 "symbol": "circle",
 300             },
 301             text=hover,
 302             hoverinfo="text+name",
 303             showlegend=True,
 304             legendgroup=name,
 305             customdata=customdata
 306         ),
 307         go.Scatter(  # Trend line
 308             x=x_axis,
 309             y=trend_avg,
 310             name=name,
 311             mode="lines",
 312             line={
 313                 "shape": "linear",
 314                 "width": 1,
 315                 "color": color,
 316             },
 317             text=hover_trend,
 318             hoverinfo="text+name",
 319             showlegend=False,
 320             legendgroup=name,
 321         )
 322     ]
 323
 324     if anomalies:
 325         anomaly_x = list()
 326         anomaly_y = list()
 327         anomaly_color = list()
 328         hover = list()
 329         for idx, anomaly in enumerate(anomalies):
 330             if anomaly in ("regression", "progression"):
 331                 anomaly_x.append(x_axis[idx])
 332                 anomaly_y.append(trend_avg[idx])
 333                 anomaly_color.append(_ANOMALY_COLOR[anomaly])
 334                 hover_itm = (
 335                     f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 336                     f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
 337                     f"classification: {anomaly}"
 338                 )
 339                 if ttype == "pdr-lat":
 340                     hover_itm = hover_itm.replace("[pps]", "[us]")
 341                 hover.append(hover_itm)
 342         anomaly_color.extend([0.0, 0.5, 1.0])
 343         traces.append(
 344             go.Scatter(
 345                 x=anomaly_x,
 346                 y=anomaly_y,
 347                 mode="markers",
 348                 text=hover,
 349                 hoverinfo="text+name",
 350                 showlegend=False,
 351                 legendgroup=name,
 352                 name=name,
 353                 marker={
 354                     "size": 15,
 355                     "symbol": "circle-open",
 356                     "color": anomaly_color,
 357                     "colorscale": _COLORSCALE_LAT \
 358                         if ttype == "pdr-lat" else _COLORSCALE_TPUT,
 359                     "showscale": True,
 360                     "line": {
 361                         "width": 2
 362                     },
 363                     "colorbar": {
 364                         "y": 0.5,
 365                         "len": 0.8,
 366                         "title": "Circles Marking Data Classification",
 367                         "titleside": "right",
 368                         "tickmode": "array",
 369                         "tickvals": [0.167, 0.500, 0.833],
 370                         "ticktext": _TICK_TEXT_LAT \
 371                             if ttype == "pdr-lat" else _TICK_TEXT_TPUT,
 372                         "ticks": "",
 373                         "ticklen": 0,
 374                         "tickangle": -90,
 375                         "thickness": 10
 376                     }
 377                 }
 378             )
 379         )
 380
 381     return traces
 382
 383
 384 def graph_trending(data: pd.DataFrame, sel:dict, layout: dict,
 385     start: datetime, end: datetime, normalize: bool) -> tuple:
 386     """
 387     """
 388
 389     if not sel:
 390         return None, None
 391
 392     fig_tput = None
 393     fig_lat = None
 394     for idx, itm in enumerate(sel):
 395
 396         df = select_trending_data(data, itm)
 397         if df is None or df.empty:
 398             continue
 399
 400         name = "-".join((itm["dut"], itm["phy"], itm["framesize"], itm["core"],
 401             itm["test"], itm["testtype"], ))
 402         if normalize:
 403             phy = itm["phy"].split("-")
 404             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 405             norm_factor = (_NORM_FREQUENCY / _FREQURENCY[topo_arch]) \
 406                 if topo_arch else 1.0
 407         else:
 408             norm_factor = 1.0
 409         traces = _generate_trending_traces(
 410             itm["testtype"], name, df, start, end, _get_color(idx), norm_factor
 411         )
 412         if traces:
 413             if not fig_tput:
 414                 fig_tput = go.Figure()
 415             fig_tput.add_traces(traces)
 416
 417         if itm["testtype"] == "pdr":
 418             traces = _generate_trending_traces(
 419                 "pdr-lat", name, df, start, end, _get_color(idx), norm_factor
 420             )
 421             if traces:
 422                 if not fig_lat:
 423                     fig_lat = go.Figure()
 424                 fig_lat.add_traces(traces)
 425
 426     if fig_tput:
 427         fig_tput.update_layout(layout.get("plot-trending-tput", dict()))
 428     if fig_lat:
 429         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 430
 431     return fig_tput, fig_lat
 432
 433
 434 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 435     """
 436     """
 437
 438     fig = None
 439
 440     traces = list()
 441     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
 442         try:
 443             decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
 444         except (hdrh.codec.HdrLengthException, TypeError) as err:
 445             continue
 446         previous_x = 0.0
 447         prev_perc = 0.0
 448         xaxis = list()
 449         yaxis = list()
 450         hovertext = list()
 451         for item in decoded.get_recorded_iterator():
 452             # The real value is "percentile".
 453             # For 100%, we cut that down to "x_perc" to avoid
 454             # infinity.
 455             percentile = item.percentile_level_iterated_to
 456             x_perc = min(percentile, PERCENTILE_MAX)
 457             xaxis.append(previous_x)
 458             yaxis.append(item.value_iterated_to)
 459             hovertext.append(
 460                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 461                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 462                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 463                 f"Latency: {item.value_iterated_to}uSec"
 464             )
 465             next_x = 100.0 / (100.0 - x_perc)
 466             xaxis.append(next_x)
 467             yaxis.append(item.value_iterated_to)
 468             hovertext.append(
 469                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 470                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 471                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 472                 f"Latency: {item.value_iterated_to}uSec"
 473             )
 474             previous_x = next_x
 475             prev_perc = percentile
 476
 477         traces.append(
 478             go.Scatter(
 479                 x=xaxis,
 480                 y=yaxis,
 481                 name=_GRAPH_LAT_HDRH_DESC[lat_name],
 482                 mode="lines",
 483                 legendgroup=_GRAPH_LAT_HDRH_DESC[lat_name],
 484                 showlegend=bool(idx % 2),
 485                 line=dict(
 486                     color=_get_color(int(idx/2)),
 487                     dash="solid",
 488                     width=1 if idx % 2 else 2
 489                 ),
 490                 hovertext=hovertext,
 491                 hoverinfo="text"
 492             )
 493         )
 494     if traces:
 495         fig = go.Figure()
 496         fig.add_traces(traces)
 497         layout_hdrh = layout.get("plot-hdrh-latency", None)
 498         if lat_hdrh:
 499             fig.update_layout(layout_hdrh)
 500
 501     return fig