resources/tools/dash/app/pal/trending/graphs.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """
  15 """
  16
  17 import plotly.graph_objects as go
  18 import pandas as pd
  19 import re
  20
  21 import hdrh.histogram
  22 import hdrh.codec
  23
  24 from datetime import datetime
  25 from numpy import isnan
  26
  27 from ..jumpavg import classify
  28
  29
  30 _COLORS = (
  31     u"#1A1110", u"#DA2647", u"#214FC6", u"#01786F", u"#BD8260", u"#FFD12A",
  32     u"#A6E7FF", u"#738276", u"#C95A49", u"#FC5A8D", u"#CEC8EF", u"#391285",
  33     u"#6F2DA8", u"#FF878D", u"#45A27D", u"#FFD0B9", u"#FD5240", u"#DB91EF",
  34     u"#44D7A8", u"#4F86F7", u"#84DE02", u"#FFCFF1", u"#614051"
  35 )
  36 _ANOMALY_COLOR = {
  37     u"regression": 0.0,
  38     u"normal": 0.5,
  39     u"progression": 1.0
  40 }
  41 _COLORSCALE_TPUT = [
  42     [0.00, u"red"],
  43     [0.33, u"red"],
  44     [0.33, u"white"],
  45     [0.66, u"white"],
  46     [0.66, u"green"],
  47     [1.00, u"green"]
  48 ]
  49 _TICK_TEXT_TPUT = [u"Regression", u"Normal", u"Progression"]
  50 _COLORSCALE_LAT = [
  51     [0.00, u"green"],
  52     [0.33, u"green"],
  53     [0.33, u"white"],
  54     [0.66, u"white"],
  55     [0.66, u"red"],
  56     [1.00, u"red"]
  57 ]
  58 _TICK_TEXT_LAT = [u"Progression", u"Normal", u"Regression"]
  59 _VALUE = {
  60     "mrr": "result_receive_rate_rate_avg",
  61     "ndr": "result_ndr_lower_rate_value",
  62     "pdr": "result_pdr_lower_rate_value",
  63     "pdr-lat": "result_latency_forward_pdr_50_avg"
  64 }
  65 _UNIT = {
  66     "mrr": "result_receive_rate_rate_unit",
  67     "ndr": "result_ndr_lower_rate_unit",
  68     "pdr": "result_pdr_lower_rate_unit",
  69     "pdr-lat": "result_latency_forward_pdr_50_unit"
  70 }
  71 _LAT_HDRH = (  # Do not change the order
  72     "result_latency_forward_pdr_0_hdrh",
  73     "result_latency_reverse_pdr_0_hdrh",
  74     "result_latency_forward_pdr_10_hdrh",
  75     "result_latency_reverse_pdr_10_hdrh",
  76     "result_latency_forward_pdr_50_hdrh",
  77     "result_latency_reverse_pdr_50_hdrh",
  78     "result_latency_forward_pdr_90_hdrh",
  79     "result_latency_reverse_pdr_90_hdrh",
  80 )
  81 # This value depends on latency stream rate (9001 pps) and duration (5s).
  82 # Keep it slightly higher to ensure rounding errors to not remove tick mark.
  83 PERCENTILE_MAX = 99.999501
  84
  85 _GRAPH_LAT_HDRH_DESC = {
  86     u"result_latency_forward_pdr_0_hdrh": u"No-load.",
  87     u"result_latency_reverse_pdr_0_hdrh": u"No-load.",
  88     u"result_latency_forward_pdr_10_hdrh": u"Low-load, 10% PDR.",
  89     u"result_latency_reverse_pdr_10_hdrh": u"Low-load, 10% PDR.",
  90     u"result_latency_forward_pdr_50_hdrh": u"Mid-load, 50% PDR.",
  91     u"result_latency_reverse_pdr_50_hdrh": u"Mid-load, 50% PDR.",
  92     u"result_latency_forward_pdr_90_hdrh": u"High-load, 90% PDR.",
  93     u"result_latency_reverse_pdr_90_hdrh": u"High-load, 90% PDR."
  94 }
  95
  96
  97 def _get_hdrh_latencies(row: pd.Series, name: str) -> dict:
  98     """
  99     """
 100
 101     latencies = {"name": name}
 102     for key in _LAT_HDRH:
 103         try:
 104             latencies[key] = row[key]
 105         except KeyError:
 106             return None
 107
 108     return latencies
 109
 110
 111 def _classify_anomalies(data):
 112     """Process the data and return anomalies and trending values.
 113
 114     Gather data into groups with average as trend value.
 115     Decorate values within groups to be normal,
 116     the first value of changed average as a regression, or a progression.
 117
 118     :param data: Full data set with unavailable samples replaced by nan.
 119     :type data: OrderedDict
 120     :returns: Classification and trend values
 121     :rtype: 3-tuple, list of strings, list of floats and list of floats
 122     """
 123     # NaN means something went wrong.
 124     # Use 0.0 to cause that being reported as a severe regression.
 125     bare_data = [0.0 if isnan(sample) else sample for sample in data.values()]
 126     # TODO: Make BitCountingGroupList a subclass of list again?
 127     group_list = classify(bare_data).group_list
 128     group_list.reverse()  # Just to use .pop() for FIFO.
 129     classification = list()
 130     avgs = list()
 131     stdevs = list()
 132     active_group = None
 133     values_left = 0
 134     avg = 0.0
 135     stdv = 0.0
 136     for sample in data.values():
 137         if isnan(sample):
 138             classification.append(u"outlier")
 139             avgs.append(sample)
 140             stdevs.append(sample)
 141             continue
 142         if values_left < 1 or active_group is None:
 143             values_left = 0
 144             while values_left < 1:  # Ignore empty groups (should not happen).
 145                 active_group = group_list.pop()
 146                 values_left = len(active_group.run_list)
 147             avg = active_group.stats.avg
 148             stdv = active_group.stats.stdev
 149             classification.append(active_group.comment)
 150             avgs.append(avg)
 151             stdevs.append(stdv)
 152             values_left -= 1
 153             continue
 154         classification.append(u"normal")
 155         avgs.append(avg)
 156         stdevs.append(stdv)
 157         values_left -= 1
 158     return classification, avgs, stdevs
 159
 160
 161 def select_trending_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
 162     """
 163     """
 164
 165     phy = itm["phy"].split("-")
 166     if len(phy) == 4:
 167         topo, arch, nic, drv = phy
 168         if drv == "dpdk":
 169             drv = ""
 170         else:
 171             drv += "-"
 172             drv = drv.replace("_", "-")
 173     else:
 174         return None
 175     cadence = \
 176         "weekly" if (arch == "aws" or itm["testtype"] != "mrr") else "daily"
 177     sel_topo_arch = (
 178         f"csit-vpp-perf-"
 179         f"{itm['testtype'] if itm['testtype'] == 'mrr' else 'ndrpdr'}-"
 180         f"{cadence}-master-{topo}-{arch}"
 181     )
 182     df_sel = data.loc[(data["job"] == sel_topo_arch)]
 183     regex = (
 184         f"^.*{nic}.*\.{itm['framesize']}-{itm['core']}-{drv}{itm['test']}-"
 185         f"{'mrr' if itm['testtype'] == 'mrr' else 'ndrpdr'}$"
 186     )
 187     df = df_sel.loc[
 188         df_sel["test_id"].apply(
 189             lambda x: True if re.search(regex, x) else False
 190         )
 191     ].sort_values(by="start_time", ignore_index=True)
 192
 193     return df
 194
 195
 196 def _generate_trending_traces(ttype: str, name: str, df: pd.DataFrame,
 197     start: datetime, end: datetime, color: str) -> list:
 198     """
 199     """
 200
 201     df = df.dropna(subset=[_VALUE[ttype], ])
 202     if df.empty:
 203         return list()
 204
 205     x_axis = [d for d in df["start_time"] if d >= start and d <= end]
 206     if not x_axis:
 207         return list()
 208
 209     anomalies, trend_avg, trend_stdev = _classify_anomalies(
 210         {k: v for k, v in zip(x_axis, df[_VALUE[ttype]])}
 211     )
 212
 213     hover = list()
 214     customdata = list()
 215     for _, row in df.iterrows():
 216         hover_itm = (
 217             f"date: {row['start_time'].strftime('%d-%m-%Y %H:%M:%S')}<br>"
 218             f"<prop> [{row[_UNIT[ttype]]}]: {row[_VALUE[ttype]]}<br>"
 219             f"<stdev>"
 220             f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 221             f"csit-ref: {row['job']}/{row['build']}<br>"
 222             f"hosts: {', '.join(row['hosts'])}"
 223         )
 224         if ttype == "mrr":
 225             stdev = (
 226                 f"stdev [{row['result_receive_rate_rate_unit']}]: "
 227                 f"{row['result_receive_rate_rate_stdev']}<br>"
 228             )
 229         else:
 230             stdev = ""
 231         hover_itm = hover_itm.replace(
 232             "<prop>", "latency" if ttype == "pdr-lat" else "average"
 233         ).replace("<stdev>", stdev)
 234         hover.append(hover_itm)
 235         if ttype == "pdr-lat":
 236             customdata.append(_get_hdrh_latencies(row, name))
 237
 238     hover_trend = list()
 239     for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 240         hover_itm = (
 241             f"date: {row['start_time'].strftime('%d-%m-%Y %H:%M:%S')}<br>"
 242             f"trend [pps]: {avg}<br>"
 243             f"stdev [pps]: {stdev}<br>"
 244             f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 245             f"csit-ref: {row['job']}/{row['build']}<br>"
 246             f"hosts: {', '.join(row['hosts'])}"
 247         )
 248         if ttype == "pdr-lat":
 249             hover_itm = hover_itm.replace("[pps]", "[us]")
 250         hover_trend.append(hover_itm)
 251
 252     traces = [
 253         go.Scatter(  # Samples
 254             x=x_axis,
 255             y=df[_VALUE[ttype]],
 256             name=name,
 257             mode="markers",
 258             marker={
 259                 u"size": 5,
 260                 u"color": color,
 261                 u"symbol": u"circle",
 262             },
 263             text=hover,
 264             hoverinfo=u"text+name",
 265             showlegend=True,
 266             legendgroup=name,
 267             customdata=customdata
 268         ),
 269         go.Scatter(  # Trend line
 270             x=x_axis,
 271             y=trend_avg,
 272             name=name,
 273             mode="lines",
 274             line={
 275                 u"shape": u"linear",
 276                 u"width": 1,
 277                 u"color": color,
 278             },
 279             text=hover_trend,
 280             hoverinfo=u"text+name",
 281             showlegend=False,
 282             legendgroup=name,
 283         )
 284     ]
 285
 286     if anomalies:
 287         anomaly_x = list()
 288         anomaly_y = list()
 289         anomaly_color = list()
 290         hover = list()
 291         for idx, anomaly in enumerate(anomalies):
 292             if anomaly in (u"regression", u"progression"):
 293                 anomaly_x.append(x_axis[idx])
 294                 anomaly_y.append(trend_avg[idx])
 295                 anomaly_color.append(_ANOMALY_COLOR[anomaly])
 296                 hover_itm = (
 297                     f"date: {x_axis[idx].strftime('%d-%m-%Y %H:%M:%S')}<br>"
 298                     f"trend [pps]: {trend_avg[idx]}<br>"
 299                     f"classification: {anomaly}"
 300                 )
 301                 if ttype == "pdr-lat":
 302                     hover_itm = hover_itm.replace("[pps]", "[us]")
 303                 hover.append(hover_itm)
 304         anomaly_color.extend([0.0, 0.5, 1.0])
 305         traces.append(
 306             go.Scatter(
 307                 x=anomaly_x,
 308                 y=anomaly_y,
 309                 mode=u"markers",
 310                 text=hover,
 311                 hoverinfo=u"text+name",
 312                 showlegend=False,
 313                 legendgroup=name,
 314                 name=name,
 315                 marker={
 316                     u"size": 15,
 317                     u"symbol": u"circle-open",
 318                     u"color": anomaly_color,
 319                     u"colorscale": _COLORSCALE_LAT \
 320                         if ttype == "pdr-lat" else _COLORSCALE_TPUT,
 321                     u"showscale": True,
 322                     u"line": {
 323                         u"width": 2
 324                     },
 325                     u"colorbar": {
 326                         u"y": 0.5,
 327                         u"len": 0.8,
 328                         u"title": u"Circles Marking Data Classification",
 329                         u"titleside": u"right",
 330                         # u"titlefont": {
 331                         #     u"size": 14
 332                         # },
 333                         u"tickmode": u"array",
 334                         u"tickvals": [0.167, 0.500, 0.833],
 335                         u"ticktext": _TICK_TEXT_LAT \
 336                             if ttype == "pdr-lat" else _TICK_TEXT_TPUT,
 337                         u"ticks": u"",
 338                         u"ticklen": 0,
 339                         u"tickangle": -90,
 340                         u"thickness": 10
 341                     }
 342                 }
 343             )
 344         )
 345
 346     return traces
 347
 348
 349 def graph_trending(data: pd.DataFrame, sel:dict, layout: dict,
 350     start: datetime, end: datetime) -> tuple:
 351     """
 352     """
 353
 354     if not sel:
 355         return None, None
 356
 357     fig_tput = None
 358     fig_lat = None
 359     for idx, itm in enumerate(sel):
 360
 361         df = select_trending_data(data, itm)
 362         if df is None:
 363             continue
 364
 365         name = (
 366             f"{itm['phy']}-{itm['framesize']}-{itm['core']}-"
 367             f"{itm['test']}-{itm['testtype']}"
 368         )
 369
 370         traces = _generate_trending_traces(
 371             itm["testtype"], name, df, start, end, _COLORS[idx % len(_COLORS)]
 372         )
 373         if traces:
 374             if not fig_tput:
 375                 fig_tput = go.Figure()
 376             fig_tput.add_traces(traces)
 377
 378         if itm["testtype"] == "pdr":
 379             traces = _generate_trending_traces(
 380                 "pdr-lat", name, df, start, end, _COLORS[idx % len(_COLORS)]
 381             )
 382             if traces:
 383                 if not fig_lat:
 384                     fig_lat = go.Figure()
 385                 fig_lat.add_traces(traces)
 386
 387     if fig_tput:
 388         fig_tput.update_layout(layout.get("plot-trending-tput", dict()))
 389     if fig_lat:
 390         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 391
 392     return fig_tput, fig_lat
 393
 394
 395 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 396     """
 397     """
 398
 399     fig = None
 400
 401     traces = list()
 402     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
 403         try:
 404             decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
 405         except (hdrh.codec.HdrLengthException, TypeError) as err:
 406             continue
 407         previous_x = 0.0
 408         prev_perc = 0.0
 409         xaxis = list()
 410         yaxis = list()
 411         hovertext = list()
 412         for item in decoded.get_recorded_iterator():
 413             # The real value is "percentile".
 414             # For 100%, we cut that down to "x_perc" to avoid
 415             # infinity.
 416             percentile = item.percentile_level_iterated_to
 417             x_perc = min(percentile, PERCENTILE_MAX)
 418             xaxis.append(previous_x)
 419             yaxis.append(item.value_iterated_to)
 420             hovertext.append(
 421                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 422                 f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
 423                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 424                 f"Latency: {item.value_iterated_to}uSec"
 425             )
 426             next_x = 100.0 / (100.0 - x_perc)
 427             xaxis.append(next_x)
 428             yaxis.append(item.value_iterated_to)
 429             hovertext.append(
 430                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 431                 f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
 432                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 433                 f"Latency: {item.value_iterated_to}uSec"
 434             )
 435             previous_x = next_x
 436             prev_perc = percentile
 437
 438         traces.append(
 439             go.Scatter(
 440                 x=xaxis,
 441                 y=yaxis,
 442                 name=_GRAPH_LAT_HDRH_DESC[lat_name],
 443                 mode=u"lines",
 444                 legendgroup=_GRAPH_LAT_HDRH_DESC[lat_name],
 445                 showlegend=bool(idx % 2),
 446                 line=dict(
 447                     color=_COLORS[int(idx/2)],
 448                     dash=u"solid",
 449                     width=1 if idx % 2 else 2
 450                 ),
 451                 hovertext=hovertext,
 452                 hoverinfo=u"text"
 453             )
 454         )
 455     if traces:
 456         fig = go.Figure()
 457         fig.add_traces(traces)
 458         layout_hdrh = layout.get("plot-hdrh-latency", None)
 459         if lat_hdrh:
 460             fig.update_layout(layout_hdrh)
 461
 462     return fig