resources/tools/dash/app/pal/trending/graphs.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """
  15 """
  16
  17
  18 import plotly.graph_objects as go
  19 import pandas as pd
  20 import re
  21
  22 from datetime import datetime
  23 from numpy import isnan
  24
  25 from ..jumpavg import classify
  26
  27
  28 _COLORS = (
  29     u"#1A1110",
  30     u"#DA2647",
  31     u"#214FC6",
  32     u"#01786F",
  33     u"#BD8260",
  34     u"#FFD12A",
  35     u"#A6E7FF",
  36     u"#738276",
  37     u"#C95A49",
  38     u"#FC5A8D",
  39     u"#CEC8EF",
  40     u"#391285",
  41     u"#6F2DA8",
  42     u"#FF878D",
  43     u"#45A27D",
  44     u"#FFD0B9",
  45     u"#FD5240",
  46     u"#DB91EF",
  47     u"#44D7A8",
  48     u"#4F86F7",
  49     u"#84DE02",
  50     u"#FFCFF1",
  51     u"#614051"
  52 )
  53 _ANOMALY_COLOR = {
  54     u"regression": 0.0,
  55     u"normal": 0.5,
  56     u"progression": 1.0
  57 }
  58 _COLORSCALE_TPUT = [
  59     [0.00, u"red"],
  60     [0.33, u"red"],
  61     [0.33, u"white"],
  62     [0.66, u"white"],
  63     [0.66, u"green"],
  64     [1.00, u"green"]
  65 ]
  66 _TICK_TEXT_TPUT = [u"Regression", u"Normal", u"Progression"]
  67 _COLORSCALE_LAT = [
  68     [0.00, u"green"],
  69     [0.33, u"green"],
  70     [0.33, u"white"],
  71     [0.66, u"white"],
  72     [0.66, u"red"],
  73     [1.00, u"red"]
  74 ]
  75 _TICK_TEXT_LAT = [u"Progression", u"Normal", u"Regression"]
  76 _VALUE = {
  77     "mrr": "result_receive_rate_rate_avg",
  78     "ndr": "result_ndr_lower_rate_value",
  79     "pdr": "result_pdr_lower_rate_value",
  80     "pdr-lat": "result_latency_forward_pdr_50_avg"
  81 }
  82 _UNIT = {
  83     "mrr": "result_receive_rate_rate_unit",
  84     "ndr": "result_ndr_lower_rate_unit",
  85     "pdr": "result_pdr_lower_rate_unit",
  86     "pdr-lat": "result_latency_forward_pdr_50_unit"
  87 }
  88
  89
  90 def _classify_anomalies(data):
  91     """Process the data and return anomalies and trending values.
  92
  93     Gather data into groups with average as trend value.
  94     Decorate values within groups to be normal,
  95     the first value of changed average as a regression, or a progression.
  96
  97     :param data: Full data set with unavailable samples replaced by nan.
  98     :type data: OrderedDict
  99     :returns: Classification and trend values
 100     :rtype: 3-tuple, list of strings, list of floats and list of floats
 101     """
 102     # NaN means something went wrong.
 103     # Use 0.0 to cause that being reported as a severe regression.
 104     bare_data = [0.0 if isnan(sample) else sample for sample in data.values()]
 105     # TODO: Make BitCountingGroupList a subclass of list again?
 106     group_list = classify(bare_data).group_list
 107     group_list.reverse()  # Just to use .pop() for FIFO.
 108     classification = list()
 109     avgs = list()
 110     stdevs = list()
 111     active_group = None
 112     values_left = 0
 113     avg = 0.0
 114     stdv = 0.0
 115     for sample in data.values():
 116         if isnan(sample):
 117             classification.append(u"outlier")
 118             avgs.append(sample)
 119             stdevs.append(sample)
 120             continue
 121         if values_left < 1 or active_group is None:
 122             values_left = 0
 123             while values_left < 1:  # Ignore empty groups (should not happen).
 124                 active_group = group_list.pop()
 125                 values_left = len(active_group.run_list)
 126             avg = active_group.stats.avg
 127             stdv = active_group.stats.stdev
 128             classification.append(active_group.comment)
 129             avgs.append(avg)
 130             stdevs.append(stdv)
 131             values_left -= 1
 132             continue
 133         classification.append(u"normal")
 134         avgs.append(avg)
 135         stdevs.append(stdv)
 136         values_left -= 1
 137     return classification, avgs, stdevs
 138
 139
 140 def trending_tput(data: pd.DataFrame, sel:dict, layout: dict, start: datetime,
 141     end: datetime):
 142     """
 143     """
 144
 145     if not sel:
 146         return None, None
 147
 148     def _generate_traces(ttype: str, name: str, df: pd.DataFrame,
 149         start: datetime, end: datetime, color: str):
 150
 151         df = df.dropna(subset=[_VALUE[ttype], ])
 152         if df.empty:
 153             return list()
 154
 155         x_axis = [d for d in df["start_time"] if d >= start and d <= end]
 156
 157         anomalies, trend_avg, trend_stdev = _classify_anomalies(
 158             {k: v for k, v in zip(x_axis, df[_VALUE[ttype]])}
 159         )
 160
 161         hover = list()
 162         for _, row in df.iterrows():
 163             hover_itm = (
 164                 f"date: {row['start_time'].strftime('%d-%m-%Y %H:%M:%S')}<br>"
 165                 f"<prop> [{row[_UNIT[ttype]]}]: {row[_VALUE[ttype]]}<br>"
 166                 f"<stdev>"
 167                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 168                 f"csit-ref: {row['job']}/{row['build']}"
 169             )
 170             if ttype == "mrr":
 171                 stdev = (
 172                     f"stdev [{row['result_receive_rate_rate_unit']}]: "
 173                     f"{row['result_receive_rate_rate_stdev']}<br>"
 174                 )
 175             else:
 176                 stdev = ""
 177             hover_itm = hover_itm.replace(
 178                 "<prop>", "latency" if ttype == "pdr-lat" else "average"
 179             ).replace("<stdev>", stdev)
 180             hover.append(hover_itm)
 181
 182         hover_trend = list()
 183         for avg, stdev in zip(trend_avg, trend_stdev):
 184             if ttype == "pdr-lat":
 185                 hover_trend.append(
 186                     f"trend [us]: {avg}<br>"
 187                     f"stdev [us]: {stdev}"
 188                 )
 189             else:
 190                 hover_trend.append(
 191                     f"trend [pps]: {avg}<br>"
 192                     f"stdev [pps]: {stdev}"
 193                 )
 194
 195         traces = [
 196             go.Scatter(  # Samples
 197                 x=x_axis,
 198                 y=df[_VALUE[ttype]],
 199                 name=name,
 200                 mode="markers",
 201                 marker={
 202                     u"size": 5,
 203                     u"color": color,
 204                     u"symbol": u"circle",
 205                 },
 206                 text=hover,
 207                 hoverinfo=u"text+name",
 208                 showlegend=True,
 209                 legendgroup=name,
 210             ),
 211             go.Scatter(  # Trend line
 212                 x=x_axis,
 213                 y=trend_avg,
 214                 name=name,
 215                 mode="lines",
 216                 line={
 217                     u"shape": u"linear",
 218                     u"width": 1,
 219                     u"color": color,
 220                 },
 221                 text=hover_trend,
 222                 hoverinfo=u"text+name",
 223                 showlegend=False,
 224                 legendgroup=name,
 225             )
 226         ]
 227
 228         if anomalies:
 229             anomaly_x = list()
 230             anomaly_y = list()
 231             anomaly_color = list()
 232             for idx, anomaly in enumerate(anomalies):
 233                 if anomaly in (u"regression", u"progression"):
 234                     anomaly_x.append(x_axis[idx])
 235                     anomaly_y.append(trend_avg[idx])
 236                     anomaly_color.append(_ANOMALY_COLOR[anomaly])
 237             anomaly_color.extend([0.0, 0.5, 1.0])
 238             traces.append(
 239                 go.Scatter(
 240                     x=anomaly_x,
 241                     y=anomaly_y,
 242                     mode=u"markers",
 243                     hoverinfo=u"none",
 244                     showlegend=False,
 245                     legendgroup=name,
 246                     name=f"{name}-anomalies",
 247                     marker={
 248                         u"size": 15,
 249                         u"symbol": u"circle-open",
 250                         u"color": anomaly_color,
 251                         u"colorscale": _COLORSCALE_LAT \
 252                             if ttype == "pdr-lat" else _COLORSCALE_TPUT,
 253                         u"showscale": True,
 254                         u"line": {
 255                             u"width": 2
 256                         },
 257                         u"colorbar": {
 258                             u"y": 0.5,
 259                             u"len": 0.8,
 260                             u"title": u"Circles Marking Data Classification",
 261                             u"titleside": u"right",
 262                             u"titlefont": {
 263                                 u"size": 14
 264                             },
 265                             u"tickmode": u"array",
 266                             u"tickvals": [0.167, 0.500, 0.833],
 267                             u"ticktext": _TICK_TEXT_LAT \
 268                                 if ttype == "pdr-lat" else _TICK_TEXT_TPUT,
 269                             u"ticks": u"",
 270                             u"ticklen": 0,
 271                             u"tickangle": -90,
 272                             u"thickness": 10
 273                         }
 274                     }
 275                 )
 276             )
 277
 278         return traces
 279
 280     # Generate graph:
 281     fig_tput = None
 282     fig_lat = None
 283     for idx, itm in enumerate(sel):
 284         phy = itm["phy"].split("-")
 285         if len(phy) == 4:
 286             topo, arch, nic, drv = phy
 287             if drv in ("dpdk", "ixgbe"):
 288                 drv = ""
 289             else:
 290                 drv += "-"
 291                 drv = drv.replace("_", "-")
 292         else:
 293             continue
 294         cadence = \
 295             "weekly" if (arch == "aws" or itm["testtype"] != "mrr") else "daily"
 296         sel_topo_arch = (
 297             f"csit-vpp-perf-"
 298             f"{itm['testtype'] if itm['testtype'] == 'mrr' else 'ndrpdr'}-"
 299             f"{cadence}-master-{topo}-{arch}"
 300         )
 301         df_sel = data.loc[(data["job"] == sel_topo_arch)]
 302         regex = (
 303             f"^.*{nic}.*\.{itm['framesize']}-{itm['core']}-{drv}{itm['test']}-"
 304             f"{'mrr' if itm['testtype'] == 'mrr' else 'ndrpdr'}$"
 305         )
 306         df = df_sel.loc[
 307             df_sel["test_id"].apply(
 308                 lambda x: True if re.search(regex, x) else False
 309             )
 310         ].sort_values(by="start_time", ignore_index=True)
 311         name = (
 312             f"{itm['phy']}-{itm['framesize']}-{itm['core']}-"
 313             f"{itm['test']}-{itm['testtype']}"
 314         )
 315
 316         traces = _generate_traces(
 317             itm["testtype"], name, df, start, end, _COLORS[idx % len(_COLORS)]
 318         )
 319         if traces:
 320             if not fig_tput:
 321                 fig_tput = go.Figure()
 322             for trace in traces:
 323                 fig_tput.add_trace(trace)
 324
 325         if itm["testtype"] == "pdr":
 326             traces = _generate_traces(
 327                 "pdr-lat", name, df, start, end, _COLORS[idx % len(_COLORS)]
 328             )
 329             if traces:
 330                 if not fig_lat:
 331                     fig_lat = go.Figure()
 332                 for trace in traces:
 333                     fig_lat.add_trace(trace)
 334
 335     if fig_tput:
 336         fig_tput.update_layout(layout.get("plot-trending-tput", dict()))
 337     if fig_lat:
 338         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 339
 340     return fig_tput, fig_lat