resources/tools/dash/app/pal/trending/graphs.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """
  15 """
  16
  17 import plotly.graph_objects as go
  18 import pandas as pd
  19
  20 import hdrh.histogram
  21 import hdrh.codec
  22
  23 from datetime import datetime
  24
  25 from ..data.utils import classify_anomalies
  26
  27 _NORM_FREQUENCY = 2.0  # [GHz]
  28 _FREQURENCY = {  # [GHz]
  29     "2n-aws": 1.000,
  30     "2n-dnv": 2.000,
  31     "2n-clx": 2.300,
  32     "2n-icx": 2.600,
  33     "2n-skx": 2.500,
  34     "2n-tx2": 2.500,
  35     "2n-zn2": 2.900,
  36     "3n-alt": 3.000,
  37     "3n-aws": 1.000,
  38     "3n-dnv": 2.000,
  39     "3n-icx": 2.600,
  40     "3n-skx": 2.500,
  41     "3n-tsh": 2.200
  42 }
  43
  44 _ANOMALY_COLOR = {
  45     "regression": 0.0,
  46     "normal": 0.5,
  47     "progression": 1.0
  48 }
  49 _COLORSCALE_TPUT = [
  50     [0.00, "red"],
  51     [0.33, "red"],
  52     [0.33, "white"],
  53     [0.66, "white"],
  54     [0.66, "green"],
  55     [1.00, "green"]
  56 ]
  57 _TICK_TEXT_TPUT = ["Regression", "Normal", "Progression"]
  58 _COLORSCALE_LAT = [
  59     [0.00, "green"],
  60     [0.33, "green"],
  61     [0.33, "white"],
  62     [0.66, "white"],
  63     [0.66, "red"],
  64     [1.00, "red"]
  65 ]
  66 _TICK_TEXT_LAT = ["Progression", "Normal", "Regression"]
  67 _VALUE = {
  68     "mrr": "result_receive_rate_rate_avg",
  69     "ndr": "result_ndr_lower_rate_value",
  70     "pdr": "result_pdr_lower_rate_value",
  71     "pdr-lat": "result_latency_forward_pdr_50_avg"
  72 }
  73 _UNIT = {
  74     "mrr": "result_receive_rate_rate_unit",
  75     "ndr": "result_ndr_lower_rate_unit",
  76     "pdr": "result_pdr_lower_rate_unit",
  77     "pdr-lat": "result_latency_forward_pdr_50_unit"
  78 }
  79 _LAT_HDRH = (  # Do not change the order
  80     "result_latency_forward_pdr_0_hdrh",
  81     "result_latency_reverse_pdr_0_hdrh",
  82     "result_latency_forward_pdr_10_hdrh",
  83     "result_latency_reverse_pdr_10_hdrh",
  84     "result_latency_forward_pdr_50_hdrh",
  85     "result_latency_reverse_pdr_50_hdrh",
  86     "result_latency_forward_pdr_90_hdrh",
  87     "result_latency_reverse_pdr_90_hdrh",
  88 )
  89 # This value depends on latency stream rate (9001 pps) and duration (5s).
  90 # Keep it slightly higher to ensure rounding errors to not remove tick mark.
  91 PERCENTILE_MAX = 99.999501
  92
  93 _GRAPH_LAT_HDRH_DESC = {
  94     "result_latency_forward_pdr_0_hdrh": "No-load.",
  95     "result_latency_reverse_pdr_0_hdrh": "No-load.",
  96     "result_latency_forward_pdr_10_hdrh": "Low-load, 10% PDR.",
  97     "result_latency_reverse_pdr_10_hdrh": "Low-load, 10% PDR.",
  98     "result_latency_forward_pdr_50_hdrh": "Mid-load, 50% PDR.",
  99     "result_latency_reverse_pdr_50_hdrh": "Mid-load, 50% PDR.",
 100     "result_latency_forward_pdr_90_hdrh": "High-load, 90% PDR.",
 101     "result_latency_reverse_pdr_90_hdrh": "High-load, 90% PDR."
 102 }
 103
 104
 105 def _get_color(idx: int) -> str:
 106     """
 107     """
 108     _COLORS = (
 109         "#1A1110", "#DA2647", "#214FC6", "#01786F", "#BD8260", "#FFD12A",
 110         "#A6E7FF", "#738276", "#C95A49", "#FC5A8D", "#CEC8EF", "#391285",
 111         "#6F2DA8", "#FF878D", "#45A27D", "#FFD0B9", "#FD5240", "#DB91EF",
 112         "#44D7A8", "#4F86F7", "#84DE02", "#FFCFF1", "#614051"
 113     )
 114     return _COLORS[idx % len(_COLORS)]
 115
 116
 117 def _get_hdrh_latencies(row: pd.Series, name: str) -> dict:
 118     """
 119     """
 120
 121     latencies = {"name": name}
 122     for key in _LAT_HDRH:
 123         try:
 124             latencies[key] = row[key]
 125         except KeyError:
 126             return None
 127
 128     return latencies
 129
 130
 131 def select_trending_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
 132     """
 133     """
 134
 135     phy = itm["phy"].split("-")
 136     if len(phy) == 4:
 137         topo, arch, nic, drv = phy
 138         if drv == "dpdk":
 139             drv = ""
 140         else:
 141             drv += "-"
 142             drv = drv.replace("_", "-")
 143     else:
 144         return None
 145
 146     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
 147     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
 148     dut_v100 = "none" if itm["dut"] == "trex" else itm["dut"]
 149     dut_v101 = itm["dut"]
 150
 151     df = data.loc[(
 152         (
 153             (
 154                 (data["version"] == "1.0.0") &
 155                 (data["dut_type"].str.lower() == dut_v100)
 156             ) |
 157             (
 158                 (data["version"] == "1.0.1") &
 159                 (data["dut_type"].str.lower() == dut_v101)
 160             )
 161         ) &
 162         (data["test_type"] == ttype) &
 163         (data["passed"] == True)
 164     )]
 165     df = df[df.job.str.endswith(f"{topo}-{arch}")]
 166     df = df[df.test_id.str.contains(
 167         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
 168         regex=True
 169     )].sort_values(by="start_time", ignore_index=True)
 170
 171     return df
 172
 173
 174 def _generate_trending_traces(ttype: str, name: str, df: pd.DataFrame,
 175     start: datetime, end: datetime, color: str, norm_factor: float) -> list:
 176     """
 177     """
 178
 179     df = df.dropna(subset=[_VALUE[ttype], ])
 180     if df.empty:
 181         return list()
 182     df = df.loc[((df["start_time"] >= start) & (df["start_time"] <= end))]
 183     if df.empty:
 184         return list()
 185
 186     x_axis = df["start_time"].tolist()
 187     if ttype == "pdr-lat":
 188         y_data = [(itm / norm_factor) for itm in df[_VALUE[ttype]].tolist()]
 189     else:
 190         y_data = [(itm * norm_factor) for itm in df[_VALUE[ttype]].tolist()]
 191
 192     anomalies, trend_avg, trend_stdev = classify_anomalies(
 193         {k: v for k, v in zip(x_axis, y_data)}
 194     )
 195
 196     hover = list()
 197     customdata = list()
 198     for idx, (_, row) in enumerate(df.iterrows()):
 199         d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 200         hover_itm = (
 201             f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 202             f"<prop> [{row[_UNIT[ttype]]}]: {y_data[idx]:,.0f}<br>"
 203             f"<stdev>"
 204             f"{d_type}-ref: {row['dut_version']}<br>"
 205             f"csit-ref: {row['job']}/{row['build']}<br>"
 206             f"hosts: {', '.join(row['hosts'])}"
 207         )
 208         if ttype == "mrr":
 209             stdev = (
 210                 f"stdev [{row['result_receive_rate_rate_unit']}]: "
 211                 f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 212             )
 213         else:
 214             stdev = ""
 215         hover_itm = hover_itm.replace(
 216             "<prop>", "latency" if ttype == "pdr-lat" else "average"
 217         ).replace("<stdev>", stdev)
 218         hover.append(hover_itm)
 219         if ttype == "pdr-lat":
 220             customdata.append(_get_hdrh_latencies(row, name))
 221
 222     hover_trend = list()
 223     for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 224         d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
 225         hover_itm = (
 226             f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 227             f"trend [pps]: {avg:,.0f}<br>"
 228             f"stdev [pps]: {stdev:,.0f}<br>"
 229             f"{d_type}-ref: {row['dut_version']}<br>"
 230             f"csit-ref: {row['job']}/{row['build']}<br>"
 231             f"hosts: {', '.join(row['hosts'])}"
 232         )
 233         if ttype == "pdr-lat":
 234             hover_itm = hover_itm.replace("[pps]", "[us]")
 235         hover_trend.append(hover_itm)
 236
 237     traces = [
 238         go.Scatter(  # Samples
 239             x=x_axis,
 240             y=y_data,
 241             name=name,
 242             mode="markers",
 243             marker={
 244                 "size": 5,
 245                 "color": color,
 246                 "symbol": "circle",
 247             },
 248             text=hover,
 249             hoverinfo="text+name",
 250             showlegend=True,
 251             legendgroup=name,
 252             customdata=customdata
 253         ),
 254         go.Scatter(  # Trend line
 255             x=x_axis,
 256             y=trend_avg,
 257             name=name,
 258             mode="lines",
 259             line={
 260                 "shape": "linear",
 261                 "width": 1,
 262                 "color": color,
 263             },
 264             text=hover_trend,
 265             hoverinfo="text+name",
 266             showlegend=False,
 267             legendgroup=name,
 268         )
 269     ]
 270
 271     if anomalies:
 272         anomaly_x = list()
 273         anomaly_y = list()
 274         anomaly_color = list()
 275         hover = list()
 276         for idx, anomaly in enumerate(anomalies):
 277             if anomaly in ("regression", "progression"):
 278                 anomaly_x.append(x_axis[idx])
 279                 anomaly_y.append(trend_avg[idx])
 280                 anomaly_color.append(_ANOMALY_COLOR[anomaly])
 281                 hover_itm = (
 282                     f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 283                     f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
 284                     f"classification: {anomaly}"
 285                 )
 286                 if ttype == "pdr-lat":
 287                     hover_itm = hover_itm.replace("[pps]", "[us]")
 288                 hover.append(hover_itm)
 289         anomaly_color.extend([0.0, 0.5, 1.0])
 290         traces.append(
 291             go.Scatter(
 292                 x=anomaly_x,
 293                 y=anomaly_y,
 294                 mode="markers",
 295                 text=hover,
 296                 hoverinfo="text+name",
 297                 showlegend=False,
 298                 legendgroup=name,
 299                 name=name,
 300                 marker={
 301                     "size": 15,
 302                     "symbol": "circle-open",
 303                     "color": anomaly_color,
 304                     "colorscale": _COLORSCALE_LAT \
 305                         if ttype == "pdr-lat" else _COLORSCALE_TPUT,
 306                     "showscale": True,
 307                     "line": {
 308                         "width": 2
 309                     },
 310                     "colorbar": {
 311                         "y": 0.5,
 312                         "len": 0.8,
 313                         "title": "Circles Marking Data Classification",
 314                         "titleside": "right",
 315                         "tickmode": "array",
 316                         "tickvals": [0.167, 0.500, 0.833],
 317                         "ticktext": _TICK_TEXT_LAT \
 318                             if ttype == "pdr-lat" else _TICK_TEXT_TPUT,
 319                         "ticks": "",
 320                         "ticklen": 0,
 321                         "tickangle": -90,
 322                         "thickness": 10
 323                     }
 324                 }
 325             )
 326         )
 327
 328     return traces
 329
 330
 331 def graph_trending(data: pd.DataFrame, sel:dict, layout: dict,
 332     start: datetime, end: datetime, normalize: bool) -> tuple:
 333     """
 334     """
 335
 336     if not sel:
 337         return None, None
 338
 339     fig_tput = None
 340     fig_lat = None
 341     for idx, itm in enumerate(sel):
 342
 343         df = select_trending_data(data, itm)
 344         if df is None or df.empty:
 345             continue
 346
 347         name = "-".join((itm["dut"], itm["phy"], itm["framesize"], itm["core"],
 348             itm["test"], itm["testtype"], ))
 349         if normalize:
 350             phy = itm["phy"].split("-")
 351             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 352             norm_factor = (_NORM_FREQUENCY / _FREQURENCY[topo_arch]) \
 353                 if topo_arch else 1.0
 354         else:
 355             norm_factor = 1.0
 356         traces = _generate_trending_traces(
 357             itm["testtype"], name, df, start, end, _get_color(idx), norm_factor
 358         )
 359         if traces:
 360             if not fig_tput:
 361                 fig_tput = go.Figure()
 362             fig_tput.add_traces(traces)
 363
 364         if itm["testtype"] == "pdr":
 365             traces = _generate_trending_traces(
 366                 "pdr-lat", name, df, start, end, _get_color(idx), norm_factor
 367             )
 368             if traces:
 369                 if not fig_lat:
 370                     fig_lat = go.Figure()
 371                 fig_lat.add_traces(traces)
 372
 373     if fig_tput:
 374         fig_tput.update_layout(layout.get("plot-trending-tput", dict()))
 375     if fig_lat:
 376         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 377
 378     return fig_tput, fig_lat
 379
 380
 381 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 382     """
 383     """
 384
 385     fig = None
 386
 387     traces = list()
 388     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
 389         try:
 390             decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
 391         except (hdrh.codec.HdrLengthException, TypeError) as err:
 392             continue
 393         previous_x = 0.0
 394         prev_perc = 0.0
 395         xaxis = list()
 396         yaxis = list()
 397         hovertext = list()
 398         for item in decoded.get_recorded_iterator():
 399             # The real value is "percentile".
 400             # For 100%, we cut that down to "x_perc" to avoid
 401             # infinity.
 402             percentile = item.percentile_level_iterated_to
 403             x_perc = min(percentile, PERCENTILE_MAX)
 404             xaxis.append(previous_x)
 405             yaxis.append(item.value_iterated_to)
 406             hovertext.append(
 407                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 408                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 409                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 410                 f"Latency: {item.value_iterated_to}uSec"
 411             )
 412             next_x = 100.0 / (100.0 - x_perc)
 413             xaxis.append(next_x)
 414             yaxis.append(item.value_iterated_to)
 415             hovertext.append(
 416                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 417                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 418                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 419                 f"Latency: {item.value_iterated_to}uSec"
 420             )
 421             previous_x = next_x
 422             prev_perc = percentile
 423
 424         traces.append(
 425             go.Scatter(
 426                 x=xaxis,
 427                 y=yaxis,
 428                 name=_GRAPH_LAT_HDRH_DESC[lat_name],
 429                 mode="lines",
 430                 legendgroup=_GRAPH_LAT_HDRH_DESC[lat_name],
 431                 showlegend=bool(idx % 2),
 432                 line=dict(
 433                     color=_get_color(int(idx/2)),
 434                     dash="solid",
 435                     width=1 if idx % 2 else 2
 436                 ),
 437                 hovertext=hovertext,
 438                 hoverinfo="text"
 439             )
 440         )
 441     if traces:
 442         fig = go.Figure()
 443         fig.add_traces(traces)
 444         layout_hdrh = layout.get("plot-hdrh-latency", None)
 445         if lat_hdrh:
 446             fig.update_layout(layout_hdrh)
 447
 448     return fig