resources/tools/dash/app/pal/trending/graphs.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """
  15 """
  16
  17 import plotly.graph_objects as go
  18 import pandas as pd
  19 import re
  20
  21 import hdrh.histogram
  22 import hdrh.codec
  23
  24 from datetime import datetime
  25 from numpy import isnan
  26
  27 from ..jumpavg import classify
  28
  29
  30 _COLORS = (
  31     u"#1A1110", u"#DA2647", u"#214FC6", u"#01786F", u"#BD8260", u"#FFD12A",
  32     u"#A6E7FF", u"#738276", u"#C95A49", u"#FC5A8D", u"#CEC8EF", u"#391285",
  33     u"#6F2DA8", u"#FF878D", u"#45A27D", u"#FFD0B9", u"#FD5240", u"#DB91EF",
  34     u"#44D7A8", u"#4F86F7", u"#84DE02", u"#FFCFF1", u"#614051"
  35 )
  36 _ANOMALY_COLOR = {
  37     u"regression": 0.0,
  38     u"normal": 0.5,
  39     u"progression": 1.0
  40 }
  41 _COLORSCALE_TPUT = [
  42     [0.00, u"red"],
  43     [0.33, u"red"],
  44     [0.33, u"white"],
  45     [0.66, u"white"],
  46     [0.66, u"green"],
  47     [1.00, u"green"]
  48 ]
  49 _TICK_TEXT_TPUT = [u"Regression", u"Normal", u"Progression"]
  50 _COLORSCALE_LAT = [
  51     [0.00, u"green"],
  52     [0.33, u"green"],
  53     [0.33, u"white"],
  54     [0.66, u"white"],
  55     [0.66, u"red"],
  56     [1.00, u"red"]
  57 ]
  58 _TICK_TEXT_LAT = [u"Progression", u"Normal", u"Regression"]
  59 _VALUE = {
  60     "mrr": "result_receive_rate_rate_avg",
  61     "ndr": "result_ndr_lower_rate_value",
  62     "pdr": "result_pdr_lower_rate_value",
  63     "pdr-lat": "result_latency_forward_pdr_50_avg"
  64 }
  65 _UNIT = {
  66     "mrr": "result_receive_rate_rate_unit",
  67     "ndr": "result_ndr_lower_rate_unit",
  68     "pdr": "result_pdr_lower_rate_unit",
  69     "pdr-lat": "result_latency_forward_pdr_50_unit"
  70 }
  71 _LAT_HDRH = (  # Do not change the order
  72     "result_latency_forward_pdr_0_hdrh",
  73     "result_latency_reverse_pdr_0_hdrh",
  74     "result_latency_forward_pdr_10_hdrh",
  75     "result_latency_reverse_pdr_10_hdrh",
  76     "result_latency_forward_pdr_50_hdrh",
  77     "result_latency_reverse_pdr_50_hdrh",
  78     "result_latency_forward_pdr_90_hdrh",
  79     "result_latency_reverse_pdr_90_hdrh",
  80 )
  81 # This value depends on latency stream rate (9001 pps) and duration (5s).
  82 # Keep it slightly higher to ensure rounding errors to not remove tick mark.
  83 PERCENTILE_MAX = 99.999501
  84
  85 _GRAPH_LAT_HDRH_DESC = {
  86     u"result_latency_forward_pdr_0_hdrh": u"No-load.",
  87     u"result_latency_reverse_pdr_0_hdrh": u"No-load.",
  88     u"result_latency_forward_pdr_10_hdrh": u"Low-load, 10% PDR.",
  89     u"result_latency_reverse_pdr_10_hdrh": u"Low-load, 10% PDR.",
  90     u"result_latency_forward_pdr_50_hdrh": u"Mid-load, 50% PDR.",
  91     u"result_latency_reverse_pdr_50_hdrh": u"Mid-load, 50% PDR.",
  92     u"result_latency_forward_pdr_90_hdrh": u"High-load, 90% PDR.",
  93     u"result_latency_reverse_pdr_90_hdrh": u"High-load, 90% PDR."
  94 }
  95
  96
  97 def _get_hdrh_latencies(row: pd.Series, name: str) -> dict:
  98     """
  99     """
 100
 101     latencies = {"name": name}
 102     for key in _LAT_HDRH:
 103         try:
 104             latencies[key] = row[key]
 105         except KeyError:
 106             return None
 107
 108     return latencies
 109
 110
 111 def _classify_anomalies(data):
 112     """Process the data and return anomalies and trending values.
 113
 114     Gather data into groups with average as trend value.
 115     Decorate values within groups to be normal,
 116     the first value of changed average as a regression, or a progression.
 117
 118     :param data: Full data set with unavailable samples replaced by nan.
 119     :type data: OrderedDict
 120     :returns: Classification and trend values
 121     :rtype: 3-tuple, list of strings, list of floats and list of floats
 122     """
 123     # NaN means something went wrong.
 124     # Use 0.0 to cause that being reported as a severe regression.
 125     bare_data = [0.0 if isnan(sample) else sample for sample in data.values()]
 126     # TODO: Make BitCountingGroupList a subclass of list again?
 127     group_list = classify(bare_data).group_list
 128     group_list.reverse()  # Just to use .pop() for FIFO.
 129     classification = list()
 130     avgs = list()
 131     stdevs = list()
 132     active_group = None
 133     values_left = 0
 134     avg = 0.0
 135     stdv = 0.0
 136     for sample in data.values():
 137         if isnan(sample):
 138             classification.append(u"outlier")
 139             avgs.append(sample)
 140             stdevs.append(sample)
 141             continue
 142         if values_left < 1 or active_group is None:
 143             values_left = 0
 144             while values_left < 1:  # Ignore empty groups (should not happen).
 145                 active_group = group_list.pop()
 146                 values_left = len(active_group.run_list)
 147             avg = active_group.stats.avg
 148             stdv = active_group.stats.stdev
 149             classification.append(active_group.comment)
 150             avgs.append(avg)
 151             stdevs.append(stdv)
 152             values_left -= 1
 153             continue
 154         classification.append(u"normal")
 155         avgs.append(avg)
 156         stdevs.append(stdv)
 157         values_left -= 1
 158     return classification, avgs, stdevs
 159
 160
 161 def graph_trending_tput(data: pd.DataFrame, sel:dict, layout: dict,
 162     start: datetime, end: datetime) -> tuple:
 163     """
 164     """
 165
 166     if not sel:
 167         return None, None
 168
 169     def _generate_traces(ttype: str, name: str, df: pd.DataFrame,
 170         start: datetime, end: datetime, color: str) -> list:
 171
 172         df = df.dropna(subset=[_VALUE[ttype], ])
 173         if df.empty:
 174             return list()
 175
 176         x_axis = [d for d in df["start_time"] if d >= start and d <= end]
 177
 178         anomalies, trend_avg, trend_stdev = _classify_anomalies(
 179             {k: v for k, v in zip(x_axis, df[_VALUE[ttype]])}
 180         )
 181
 182         hover = list()
 183         customdata = list()
 184         for _, row in df.iterrows():
 185             hover_itm = (
 186                 f"date: {row['start_time'].strftime('%d-%m-%Y %H:%M:%S')}<br>"
 187                 f"<prop> [{row[_UNIT[ttype]]}]: {row[_VALUE[ttype]]}<br>"
 188                 f"<stdev>"
 189                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 190                 f"csit-ref: {row['job']}/{row['build']}"
 191             )
 192             if ttype == "mrr":
 193                 stdev = (
 194                     f"stdev [{row['result_receive_rate_rate_unit']}]: "
 195                     f"{row['result_receive_rate_rate_stdev']}<br>"
 196                 )
 197             else:
 198                 stdev = ""
 199             hover_itm = hover_itm.replace(
 200                 "<prop>", "latency" if ttype == "pdr-lat" else "average"
 201             ).replace("<stdev>", stdev)
 202             hover.append(hover_itm)
 203             if ttype == "pdr-lat":
 204                 customdata.append(_get_hdrh_latencies(row, name))
 205
 206         hover_trend = list()
 207         for avg, stdev in zip(trend_avg, trend_stdev):
 208             if ttype == "pdr-lat":
 209                 hover_trend.append(
 210                     f"trend [us]: {avg}<br>"
 211                     f"stdev [us]: {stdev}"
 212                 )
 213             else:
 214                 hover_trend.append(
 215                     f"trend [pps]: {avg}<br>"
 216                     f"stdev [pps]: {stdev}"
 217                 )
 218
 219         traces = [
 220             go.Scatter(  # Samples
 221                 x=x_axis,
 222                 y=df[_VALUE[ttype]],
 223                 name=name,
 224                 mode="markers",
 225                 marker={
 226                     u"size": 5,
 227                     u"color": color,
 228                     u"symbol": u"circle",
 229                 },
 230                 text=hover,
 231                 hoverinfo=u"text+name",
 232                 showlegend=True,
 233                 legendgroup=name,
 234                 customdata=customdata
 235             ),
 236             go.Scatter(  # Trend line
 237                 x=x_axis,
 238                 y=trend_avg,
 239                 name=name,
 240                 mode="lines",
 241                 line={
 242                     u"shape": u"linear",
 243                     u"width": 1,
 244                     u"color": color,
 245                 },
 246                 text=hover_trend,
 247                 hoverinfo=u"text+name",
 248                 showlegend=False,
 249                 legendgroup=name,
 250             )
 251         ]
 252
 253         if anomalies:
 254             anomaly_x = list()
 255             anomaly_y = list()
 256             anomaly_color = list()
 257             for idx, anomaly in enumerate(anomalies):
 258                 if anomaly in (u"regression", u"progression"):
 259                     anomaly_x.append(x_axis[idx])
 260                     anomaly_y.append(trend_avg[idx])
 261                     anomaly_color.append(_ANOMALY_COLOR[anomaly])
 262             anomaly_color.extend([0.0, 0.5, 1.0])
 263             traces.append(
 264                 go.Scatter(
 265                     x=anomaly_x,
 266                     y=anomaly_y,
 267                     mode=u"markers",
 268                     hoverinfo=u"none",
 269                     showlegend=False,
 270                     legendgroup=name,
 271                     name=f"{name}-anomalies",
 272                     marker={
 273                         u"size": 15,
 274                         u"symbol": u"circle-open",
 275                         u"color": anomaly_color,
 276                         u"colorscale": _COLORSCALE_LAT \
 277                             if ttype == "pdr-lat" else _COLORSCALE_TPUT,
 278                         u"showscale": True,
 279                         u"line": {
 280                             u"width": 2
 281                         },
 282                         u"colorbar": {
 283                             u"y": 0.5,
 284                             u"len": 0.8,
 285                             u"title": u"Circles Marking Data Classification",
 286                             u"titleside": u"right",
 287                             # u"titlefont": {
 288                             #     u"size": 14
 289                             # },
 290                             u"tickmode": u"array",
 291                             u"tickvals": [0.167, 0.500, 0.833],
 292                             u"ticktext": _TICK_TEXT_LAT \
 293                                 if ttype == "pdr-lat" else _TICK_TEXT_TPUT,
 294                             u"ticks": u"",
 295                             u"ticklen": 0,
 296                             u"tickangle": -90,
 297                             u"thickness": 10
 298                         }
 299                     }
 300                 )
 301             )
 302
 303         return traces
 304
 305     # Generate graph:
 306     fig_tput = None
 307     fig_lat = None
 308     for idx, itm in enumerate(sel):
 309         phy = itm["phy"].split("-")
 310         if len(phy) == 4:
 311             topo, arch, nic, drv = phy
 312             if drv in ("dpdk", "ixgbe"):
 313                 drv = ""
 314             else:
 315                 drv += "-"
 316                 drv = drv.replace("_", "-")
 317         else:
 318             continue
 319         cadence = \
 320             "weekly" if (arch == "aws" or itm["testtype"] != "mrr") else "daily"
 321         sel_topo_arch = (
 322             f"csit-vpp-perf-"
 323             f"{itm['testtype'] if itm['testtype'] == 'mrr' else 'ndrpdr'}-"
 324             f"{cadence}-master-{topo}-{arch}"
 325         )
 326         df_sel = data.loc[(data["job"] == sel_topo_arch)]
 327         regex = (
 328             f"^.*{nic}.*\.{itm['framesize']}-{itm['core']}-{drv}{itm['test']}-"
 329             f"{'mrr' if itm['testtype'] == 'mrr' else 'ndrpdr'}$"
 330         )
 331         df = df_sel.loc[
 332             df_sel["test_id"].apply(
 333                 lambda x: True if re.search(regex, x) else False
 334             )
 335         ].sort_values(by="start_time", ignore_index=True)
 336         name = (
 337             f"{itm['phy']}-{itm['framesize']}-{itm['core']}-"
 338             f"{itm['test']}-{itm['testtype']}"
 339         )
 340
 341         traces = _generate_traces(
 342             itm["testtype"], name, df, start, end, _COLORS[idx % len(_COLORS)]
 343         )
 344         if traces:
 345             if not fig_tput:
 346                 fig_tput = go.Figure()
 347             fig_tput.add_traces(traces)
 348
 349         if itm["testtype"] == "pdr":
 350             traces = _generate_traces(
 351                 "pdr-lat", name, df, start, end, _COLORS[idx % len(_COLORS)]
 352             )
 353             if traces:
 354                 if not fig_lat:
 355                     fig_lat = go.Figure()
 356                 fig_lat.add_traces(traces)
 357
 358     if fig_tput:
 359         fig_tput.update_layout(layout.get("plot-trending-tput", dict()))
 360     if fig_lat:
 361         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 362
 363     return fig_tput, fig_lat
 364
 365
 366 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 367     """
 368     """
 369
 370     fig = None
 371
 372     try:
 373         name = data.pop("name")
 374     except (KeyError, AttributeError):
 375         return None
 376
 377     traces = list()
 378     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
 379         try:
 380             decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
 381         except (hdrh.codec.HdrLengthException, TypeError) as err:
 382             continue
 383         previous_x = 0.0
 384         prev_perc = 0.0
 385         xaxis = list()
 386         yaxis = list()
 387         hovertext = list()
 388         for item in decoded.get_recorded_iterator():
 389             # The real value is "percentile".
 390             # For 100%, we cut that down to "x_perc" to avoid
 391             # infinity.
 392             percentile = item.percentile_level_iterated_to
 393             x_perc = min(percentile, PERCENTILE_MAX)
 394             xaxis.append(previous_x)
 395             yaxis.append(item.value_iterated_to)
 396             hovertext.append(
 397                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 398                 f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
 399                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 400                 f"Latency: {item.value_iterated_to}uSec"
 401             )
 402             next_x = 100.0 / (100.0 - x_perc)
 403             xaxis.append(next_x)
 404             yaxis.append(item.value_iterated_to)
 405             hovertext.append(
 406                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 407                 f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
 408                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 409                 f"Latency: {item.value_iterated_to}uSec"
 410             )
 411             previous_x = next_x
 412             prev_perc = percentile
 413
 414         traces.append(
 415             go.Scatter(
 416                 x=xaxis,
 417                 y=yaxis,
 418                 name=_GRAPH_LAT_HDRH_DESC[lat_name],
 419                 mode=u"lines",
 420                 legendgroup=_GRAPH_LAT_HDRH_DESC[lat_name],
 421                 showlegend=bool(idx % 2),
 422                 line=dict(
 423                     color=_COLORS[int(idx/2)],
 424                     dash=u"solid",
 425                     width=1 if idx % 2 else 2
 426                 ),
 427                 hovertext=hovertext,
 428                 hoverinfo=u"text"
 429             )
 430         )
 431     if traces:
 432         fig = go.Figure()
 433         fig.add_traces(traces)
 434         layout_hdrh = layout.get("plot-hdrh-latency", None)
 435         if lat_hdrh:
 436             layout_hdrh["title"]["text"] = f"<b>{name}</b>"
 437             fig.update_layout(layout_hdrh)
 438
 439     return fig