resources/tools/dash/app/pal/trending/graphs.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """
  15 """
  16
  17 import plotly.graph_objects as go
  18 import pandas as pd
  19 import re
  20
  21 import hdrh.histogram
  22 import hdrh.codec
  23
  24 from datetime import datetime
  25 from numpy import isnan
  26
  27 from ..jumpavg import classify
  28
  29
  30 _COLORS = (
  31     u"#1A1110", u"#DA2647", u"#214FC6", u"#01786F", u"#BD8260", u"#FFD12A",
  32     u"#A6E7FF", u"#738276", u"#C95A49", u"#FC5A8D", u"#CEC8EF", u"#391285",
  33     u"#6F2DA8", u"#FF878D", u"#45A27D", u"#FFD0B9", u"#FD5240", u"#DB91EF",
  34     u"#44D7A8", u"#4F86F7", u"#84DE02", u"#FFCFF1", u"#614051"
  35 )
  36 _ANOMALY_COLOR = {
  37     u"regression": 0.0,
  38     u"normal": 0.5,
  39     u"progression": 1.0
  40 }
  41 _COLORSCALE_TPUT = [
  42     [0.00, u"red"],
  43     [0.33, u"red"],
  44     [0.33, u"white"],
  45     [0.66, u"white"],
  46     [0.66, u"green"],
  47     [1.00, u"green"]
  48 ]
  49 _TICK_TEXT_TPUT = [u"Regression", u"Normal", u"Progression"]
  50 _COLORSCALE_LAT = [
  51     [0.00, u"green"],
  52     [0.33, u"green"],
  53     [0.33, u"white"],
  54     [0.66, u"white"],
  55     [0.66, u"red"],
  56     [1.00, u"red"]
  57 ]
  58 _TICK_TEXT_LAT = [u"Progression", u"Normal", u"Regression"]
  59 _VALUE = {
  60     "mrr": "result_receive_rate_rate_avg",
  61     "ndr": "result_ndr_lower_rate_value",
  62     "pdr": "result_pdr_lower_rate_value",
  63     "pdr-lat": "result_latency_forward_pdr_50_avg"
  64 }
  65 _UNIT = {
  66     "mrr": "result_receive_rate_rate_unit",
  67     "ndr": "result_ndr_lower_rate_unit",
  68     "pdr": "result_pdr_lower_rate_unit",
  69     "pdr-lat": "result_latency_forward_pdr_50_unit"
  70 }
  71 _LAT_HDRH = (  # Do not change the order
  72     "result_latency_forward_pdr_0_hdrh",
  73     "result_latency_reverse_pdr_0_hdrh",
  74     "result_latency_forward_pdr_10_hdrh",
  75     "result_latency_reverse_pdr_10_hdrh",
  76     "result_latency_forward_pdr_50_hdrh",
  77     "result_latency_reverse_pdr_50_hdrh",
  78     "result_latency_forward_pdr_90_hdrh",
  79     "result_latency_reverse_pdr_90_hdrh",
  80 )
  81 # This value depends on latency stream rate (9001 pps) and duration (5s).
  82 # Keep it slightly higher to ensure rounding errors to not remove tick mark.
  83 PERCENTILE_MAX = 99.999501
  84
  85 _GRAPH_LAT_HDRH_DESC = {
  86     u"result_latency_forward_pdr_0_hdrh": u"No-load.",
  87     u"result_latency_reverse_pdr_0_hdrh": u"No-load.",
  88     u"result_latency_forward_pdr_10_hdrh": u"Low-load, 10% PDR.",
  89     u"result_latency_reverse_pdr_10_hdrh": u"Low-load, 10% PDR.",
  90     u"result_latency_forward_pdr_50_hdrh": u"Mid-load, 50% PDR.",
  91     u"result_latency_reverse_pdr_50_hdrh": u"Mid-load, 50% PDR.",
  92     u"result_latency_forward_pdr_90_hdrh": u"High-load, 90% PDR.",
  93     u"result_latency_reverse_pdr_90_hdrh": u"High-load, 90% PDR."
  94 }
  95
  96
  97 def _get_hdrh_latencies(row: pd.Series, name: str) -> dict:
  98     """
  99     """
 100
 101     latencies = {"name": name}
 102     for key in _LAT_HDRH:
 103         try:
 104             latencies[key] = row[key]
 105         except KeyError:
 106             return None
 107
 108     return latencies
 109
 110
 111 def _classify_anomalies(data):
 112     """Process the data and return anomalies and trending values.
 113
 114     Gather data into groups with average as trend value.
 115     Decorate values within groups to be normal,
 116     the first value of changed average as a regression, or a progression.
 117
 118     :param data: Full data set with unavailable samples replaced by nan.
 119     :type data: OrderedDict
 120     :returns: Classification and trend values
 121     :rtype: 3-tuple, list of strings, list of floats and list of floats
 122     """
 123     # NaN means something went wrong.
 124     # Use 0.0 to cause that being reported as a severe regression.
 125     bare_data = [0.0 if isnan(sample) else sample for sample in data.values()]
 126     # TODO: Make BitCountingGroupList a subclass of list again?
 127     group_list = classify(bare_data).group_list
 128     group_list.reverse()  # Just to use .pop() for FIFO.
 129     classification = list()
 130     avgs = list()
 131     stdevs = list()
 132     active_group = None
 133     values_left = 0
 134     avg = 0.0
 135     stdv = 0.0
 136     for sample in data.values():
 137         if isnan(sample):
 138             classification.append(u"outlier")
 139             avgs.append(sample)
 140             stdevs.append(sample)
 141             continue
 142         if values_left < 1 or active_group is None:
 143             values_left = 0
 144             while values_left < 1:  # Ignore empty groups (should not happen).
 145                 active_group = group_list.pop()
 146                 values_left = len(active_group.run_list)
 147             avg = active_group.stats.avg
 148             stdv = active_group.stats.stdev
 149             classification.append(active_group.comment)
 150             avgs.append(avg)
 151             stdevs.append(stdv)
 152             values_left -= 1
 153             continue
 154         classification.append(u"normal")
 155         avgs.append(avg)
 156         stdevs.append(stdv)
 157         values_left -= 1
 158     return classification, avgs, stdevs
 159
 160
 161 def select_trending_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
 162     """
 163     """
 164
 165     phy = itm["phy"].split("-")
 166     if len(phy) == 4:
 167         topo, arch, nic, drv = phy
 168         if drv == "dpdk":
 169             drv = ""
 170         else:
 171             drv += "-"
 172             drv = drv.replace("_", "-")
 173     else:
 174         return None
 175     cadence = \
 176         "weekly" if (arch == "aws" or itm["testtype"] != "mrr") else "daily"
 177     sel_topo_arch = (
 178         f"csit-vpp-perf-"
 179         f"{itm['testtype'] if itm['testtype'] == 'mrr' else 'ndrpdr'}-"
 180         f"{cadence}-master-{topo}-{arch}"
 181     )
 182     df_sel = data.loc[(data["job"] == sel_topo_arch)]
 183     regex = (
 184         f"^.*{nic}.*\.{itm['framesize']}-{itm['core']}-{drv}{itm['test']}-"
 185         f"{'mrr' if itm['testtype'] == 'mrr' else 'ndrpdr'}$"
 186     )
 187     df = df_sel.loc[
 188         df_sel["test_id"].apply(
 189             lambda x: True if re.search(regex, x) else False
 190         )
 191     ].sort_values(by="start_time", ignore_index=True)
 192
 193     return df
 194
 195
 196 def _generate_trending_traces(ttype: str, name: str, df: pd.DataFrame,
 197     start: datetime, end: datetime, color: str) -> list:
 198     """
 199     """
 200
 201     df = df.dropna(subset=[_VALUE[ttype], ])
 202     if df.empty:
 203         return list()
 204     df = df.loc[((df["start_time"] >= start) & (df["start_time"] <= end))]
 205     if df.empty:
 206         return list()
 207
 208     x_axis = df["start_time"].tolist()
 209
 210     anomalies, trend_avg, trend_stdev = _classify_anomalies(
 211         {k: v for k, v in zip(x_axis, df[_VALUE[ttype]])}
 212     )
 213
 214     hover = list()
 215     customdata = list()
 216     for _, row in df.iterrows():
 217         hover_itm = (
 218             f"date: {row['start_time'].strftime('%d-%m-%Y %H:%M:%S')}<br>"
 219             f"<prop> [{row[_UNIT[ttype]]}]: {row[_VALUE[ttype]]}<br>"
 220             f"<stdev>"
 221             f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 222             f"csit-ref: {row['job']}/{row['build']}<br>"
 223             f"hosts: {', '.join(row['hosts'])}"
 224         )
 225         if ttype == "mrr":
 226             stdev = (
 227                 f"stdev [{row['result_receive_rate_rate_unit']}]: "
 228                 f"{row['result_receive_rate_rate_stdev']}<br>"
 229             )
 230         else:
 231             stdev = ""
 232         hover_itm = hover_itm.replace(
 233             "<prop>", "latency" if ttype == "pdr-lat" else "average"
 234         ).replace("<stdev>", stdev)
 235         hover.append(hover_itm)
 236         if ttype == "pdr-lat":
 237             customdata.append(_get_hdrh_latencies(row, name))
 238
 239     hover_trend = list()
 240     for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 241         hover_itm = (
 242             f"date: {row['start_time'].strftime('%d-%m-%Y %H:%M:%S')}<br>"
 243             f"trend [pps]: {avg}<br>"
 244             f"stdev [pps]: {stdev}<br>"
 245             f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 246             f"csit-ref: {row['job']}/{row['build']}<br>"
 247             f"hosts: {', '.join(row['hosts'])}"
 248         )
 249         if ttype == "pdr-lat":
 250             hover_itm = hover_itm.replace("[pps]", "[us]")
 251         hover_trend.append(hover_itm)
 252
 253     traces = [
 254         go.Scatter(  # Samples
 255             x=x_axis,
 256             y=df[_VALUE[ttype]],
 257             name=name,
 258             mode="markers",
 259             marker={
 260                 u"size": 5,
 261                 u"color": color,
 262                 u"symbol": u"circle",
 263             },
 264             text=hover,
 265             hoverinfo=u"text+name",
 266             showlegend=True,
 267             legendgroup=name,
 268             customdata=customdata
 269         ),
 270         go.Scatter(  # Trend line
 271             x=x_axis,
 272             y=trend_avg,
 273             name=name,
 274             mode="lines",
 275             line={
 276                 u"shape": u"linear",
 277                 u"width": 1,
 278                 u"color": color,
 279             },
 280             text=hover_trend,
 281             hoverinfo=u"text+name",
 282             showlegend=False,
 283             legendgroup=name,
 284         )
 285     ]
 286
 287     if anomalies:
 288         anomaly_x = list()
 289         anomaly_y = list()
 290         anomaly_color = list()
 291         hover = list()
 292         for idx, anomaly in enumerate(anomalies):
 293             if anomaly in (u"regression", u"progression"):
 294                 anomaly_x.append(x_axis[idx])
 295                 anomaly_y.append(trend_avg[idx])
 296                 anomaly_color.append(_ANOMALY_COLOR[anomaly])
 297                 hover_itm = (
 298                     f"date: {x_axis[idx].strftime('%d-%m-%Y %H:%M:%S')}<br>"
 299                     f"trend [pps]: {trend_avg[idx]}<br>"
 300                     f"classification: {anomaly}"
 301                 )
 302                 if ttype == "pdr-lat":
 303                     hover_itm = hover_itm.replace("[pps]", "[us]")
 304                 hover.append(hover_itm)
 305         anomaly_color.extend([0.0, 0.5, 1.0])
 306         traces.append(
 307             go.Scatter(
 308                 x=anomaly_x,
 309                 y=anomaly_y,
 310                 mode=u"markers",
 311                 text=hover,
 312                 hoverinfo=u"text+name",
 313                 showlegend=False,
 314                 legendgroup=name,
 315                 name=name,
 316                 marker={
 317                     u"size": 15,
 318                     u"symbol": u"circle-open",
 319                     u"color": anomaly_color,
 320                     u"colorscale": _COLORSCALE_LAT \
 321                         if ttype == "pdr-lat" else _COLORSCALE_TPUT,
 322                     u"showscale": True,
 323                     u"line": {
 324                         u"width": 2
 325                     },
 326                     u"colorbar": {
 327                         u"y": 0.5,
 328                         u"len": 0.8,
 329                         u"title": u"Circles Marking Data Classification",
 330                         u"titleside": u"right",
 331                         u"tickmode": u"array",
 332                         u"tickvals": [0.167, 0.500, 0.833],
 333                         u"ticktext": _TICK_TEXT_LAT \
 334                             if ttype == "pdr-lat" else _TICK_TEXT_TPUT,
 335                         u"ticks": u"",
 336                         u"ticklen": 0,
 337                         u"tickangle": -90,
 338                         u"thickness": 10
 339                     }
 340                 }
 341             )
 342         )
 343
 344     return traces
 345
 346
 347 def graph_trending(data: pd.DataFrame, sel:dict, layout: dict,
 348     start: datetime, end: datetime) -> tuple:
 349     """
 350     """
 351
 352     if not sel:
 353         return None, None
 354
 355     fig_tput = None
 356     fig_lat = None
 357     for idx, itm in enumerate(sel):
 358
 359         df = select_trending_data(data, itm)
 360         if df is None:
 361             continue
 362
 363         name = (
 364             f"{itm['phy']}-{itm['framesize']}-{itm['core']}-"
 365             f"{itm['test']}-{itm['testtype']}"
 366         )
 367
 368         traces = _generate_trending_traces(
 369             itm["testtype"], name, df, start, end, _COLORS[idx % len(_COLORS)]
 370         )
 371         if traces:
 372             if not fig_tput:
 373                 fig_tput = go.Figure()
 374             fig_tput.add_traces(traces)
 375
 376         if itm["testtype"] == "pdr":
 377             traces = _generate_trending_traces(
 378                 "pdr-lat", name, df, start, end, _COLORS[idx % len(_COLORS)]
 379             )
 380             if traces:
 381                 if not fig_lat:
 382                     fig_lat = go.Figure()
 383                 fig_lat.add_traces(traces)
 384
 385     if fig_tput:
 386         fig_tput.update_layout(layout.get("plot-trending-tput", dict()))
 387     if fig_lat:
 388         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 389
 390     return fig_tput, fig_lat
 391
 392
 393 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 394     """
 395     """
 396
 397     fig = None
 398
 399     traces = list()
 400     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
 401         try:
 402             decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
 403         except (hdrh.codec.HdrLengthException, TypeError) as err:
 404             continue
 405         previous_x = 0.0
 406         prev_perc = 0.0
 407         xaxis = list()
 408         yaxis = list()
 409         hovertext = list()
 410         for item in decoded.get_recorded_iterator():
 411             # The real value is "percentile".
 412             # For 100%, we cut that down to "x_perc" to avoid
 413             # infinity.
 414             percentile = item.percentile_level_iterated_to
 415             x_perc = min(percentile, PERCENTILE_MAX)
 416             xaxis.append(previous_x)
 417             yaxis.append(item.value_iterated_to)
 418             hovertext.append(
 419                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 420                 f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
 421                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 422                 f"Latency: {item.value_iterated_to}uSec"
 423             )
 424             next_x = 100.0 / (100.0 - x_perc)
 425             xaxis.append(next_x)
 426             yaxis.append(item.value_iterated_to)
 427             hovertext.append(
 428                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 429                 f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
 430                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 431                 f"Latency: {item.value_iterated_to}uSec"
 432             )
 433             previous_x = next_x
 434             prev_perc = percentile
 435
 436         traces.append(
 437             go.Scatter(
 438                 x=xaxis,
 439                 y=yaxis,
 440                 name=_GRAPH_LAT_HDRH_DESC[lat_name],
 441                 mode=u"lines",
 442                 legendgroup=_GRAPH_LAT_HDRH_DESC[lat_name],
 443                 showlegend=bool(idx % 2),
 444                 line=dict(
 445                     color=_COLORS[int(idx/2)],
 446                     dash=u"solid",
 447                     width=1 if idx % 2 else 2
 448                 ),
 449                 hovertext=hovertext,
 450                 hoverinfo=u"text"
 451             )
 452         )
 453     if traces:
 454         fig = go.Figure()
 455         fig.add_traces(traces)
 456         layout_hdrh = layout.get("plot-hdrh-latency", None)
 457         if lat_hdrh:
 458             fig.update_layout(layout_hdrh)
 459
 460     return fig