resources/tools/dash/app/pal/report/graphs.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """
  15 """
  16
  17 import re
  18 import plotly.graph_objects as go
  19 import pandas as pd
  20
  21 from copy import deepcopy
  22
  23 import hdrh.histogram
  24 import hdrh.codec
  25
  26
  27 _NORM_FREQUENCY = 2.0  # [GHz]
  28 _FREQURENCY = {  # [GHz]
  29     "2n-aws": 1.000,
  30     "2n-dnv": 2.000,
  31     "2n-clx": 2.300,
  32     "2n-icx": 2.600,
  33     "2n-skx": 2.500,
  34     "2n-tx2": 2.500,
  35     "2n-zn2": 2.900,
  36     "3n-alt": 3.000,
  37     "3n-aws": 1.000,
  38     "3n-dnv": 2.000,
  39     "3n-icx": 2.600,
  40     "3n-skx": 2.500,
  41     "3n-tsh": 2.200
  42 }
  43
  44 _VALUE = {
  45     "mrr": "result_receive_rate_rate_values",
  46     "ndr": "result_ndr_lower_rate_value",
  47     "pdr": "result_pdr_lower_rate_value",
  48     "pdr-lat": "result_latency_forward_pdr_50_avg"
  49 }
  50 _UNIT = {
  51     "mrr": "result_receive_rate_rate_unit",
  52     "ndr": "result_ndr_lower_rate_unit",
  53     "pdr": "result_pdr_lower_rate_unit",
  54     "pdr-lat": "result_latency_forward_pdr_50_unit"
  55 }
  56 _LAT_HDRH = (  # Do not change the order
  57     "result_latency_forward_pdr_0_hdrh",
  58     "result_latency_reverse_pdr_0_hdrh",
  59     "result_latency_forward_pdr_10_hdrh",
  60     "result_latency_reverse_pdr_10_hdrh",
  61     "result_latency_forward_pdr_50_hdrh",
  62     "result_latency_reverse_pdr_50_hdrh",
  63     "result_latency_forward_pdr_90_hdrh",
  64     "result_latency_reverse_pdr_90_hdrh",
  65 )
  66 # This value depends on latency stream rate (9001 pps) and duration (5s).
  67 # Keep it slightly higher to ensure rounding errors to not remove tick mark.
  68 PERCENTILE_MAX = 99.999501
  69
  70 _GRAPH_LAT_HDRH_DESC = {
  71     "result_latency_forward_pdr_0_hdrh": "No-load.",
  72     "result_latency_reverse_pdr_0_hdrh": "No-load.",
  73     "result_latency_forward_pdr_10_hdrh": "Low-load, 10% PDR.",
  74     "result_latency_reverse_pdr_10_hdrh": "Low-load, 10% PDR.",
  75     "result_latency_forward_pdr_50_hdrh": "Mid-load, 50% PDR.",
  76     "result_latency_reverse_pdr_50_hdrh": "Mid-load, 50% PDR.",
  77     "result_latency_forward_pdr_90_hdrh": "High-load, 90% PDR.",
  78     "result_latency_reverse_pdr_90_hdrh": "High-load, 90% PDR."
  79 }
  80
  81
  82 def _get_color(idx: int) -> str:
  83     """
  84     """
  85     _COLORS = (
  86         "#1A1110", "#DA2647", "#214FC6", "#01786F", "#BD8260", "#FFD12A",
  87         "#A6E7FF", "#738276", "#C95A49", "#FC5A8D", "#CEC8EF", "#391285",
  88         "#6F2DA8", "#FF878D", "#45A27D", "#FFD0B9", "#FD5240", "#DB91EF",
  89         "#44D7A8", "#4F86F7", "#84DE02", "#FFCFF1", "#614051"
  90     )
  91     return _COLORS[idx % len(_COLORS)]
  92
  93
  94 def get_short_version(version: str, dut_type: str="vpp") -> str:
  95     """
  96     """
  97
  98     if dut_type in ("trex", "dpdk"):
  99         return version
 100
 101     s_version = str()
 102     groups = re.search(
 103         pattern=re.compile(r"^(\d{2}).(\d{2})-(rc0|rc1|rc2|release$)"),
 104         string=version
 105     )
 106     if groups:
 107         try:
 108             s_version = \
 109                 f"{groups.group(1)}.{groups.group(2)}.{groups.group(3)}".\
 110                     replace("release", "rls")
 111         except IndexError:
 112             pass
 113
 114     return s_version
 115
 116
 117 def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
 118     """
 119     """
 120
 121     phy = itm["phy"].split("-")
 122     if len(phy) == 4:
 123         topo, arch, nic, drv = phy
 124         if drv == "dpdk":
 125             drv = ""
 126         else:
 127             drv += "-"
 128             drv = drv.replace("_", "-")
 129     else:
 130         return None
 131
 132     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
 133     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
 134     dut_v100 = "none" if itm["dut"] == "trex" else itm["dut"]
 135     dut_v101 = itm["dut"]
 136
 137     df = data.loc[(
 138         (data["release"] == itm["rls"]) &
 139         (
 140             (
 141                 (data["version"] == "1.0.0") &
 142                 (data["dut_type"].str.lower() == dut_v100)
 143             ) |
 144             (
 145                 (data["version"] == "1.0.1") &
 146                 (data["dut_type"].str.lower() == dut_v101)
 147             )
 148         ) &
 149         (data["test_type"] == ttype) &
 150         (data["passed"] == True)
 151     )]
 152     regex_test = \
 153         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$"
 154     df = df[
 155         (df.job.str.endswith(f"{topo}-{arch}")) &
 156         (df.dut_version.str.contains(itm["dutver"].replace(".r", "-r").\
 157             replace("rls", "release"))) &
 158         (df.test_id.str.contains(regex_test, regex=True))
 159     ]
 160
 161     return df
 162
 163
 164 def graph_iterative(data: pd.DataFrame, sel:dict, layout: dict,
 165         normalize: bool) -> tuple:
 166     """
 167     """
 168
 169     fig_tput = None
 170     fig_lat = None
 171
 172     tput_traces = list()
 173     y_tput_max = 0
 174     lat_traces = list()
 175     y_lat_max = 0
 176     x_lat = list()
 177     show_latency = False
 178     show_tput = False
 179     for idx, itm in enumerate(sel):
 180         itm_data = select_iterative_data(data, itm)
 181         if itm_data.empty:
 182             continue
 183         phy = itm["phy"].split("-")
 184         topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 185         norm_factor = (_NORM_FREQUENCY / _FREQURENCY[topo_arch]) \
 186             if normalize else 1.0
 187         if itm["testtype"] == "mrr":
 188             y_data_raw = itm_data[_VALUE[itm["testtype"]]].to_list()[0]
 189             y_data = [y * norm_factor for y in y_data_raw]
 190             if len(y_data) > 0:
 191                 y_tput_max = \
 192                     max(y_data) if max(y_data) > y_tput_max else y_tput_max
 193         else:
 194             y_data_raw = itm_data[_VALUE[itm["testtype"]]].to_list()
 195             y_data = [y * norm_factor for y in y_data_raw]
 196             if y_data:
 197                 y_tput_max = \
 198                     max(y_data) if max(y_data) > y_tput_max else y_tput_max
 199         nr_of_samples = len(y_data)
 200         tput_kwargs = dict(
 201             y=y_data,
 202             name=(
 203                 f"{idx + 1}. "
 204                 f"({nr_of_samples:02d} "
 205                 f"run{'s' if nr_of_samples > 1 else ''}) "
 206                 f"{itm['id']}"
 207             ),
 208             hoverinfo=u"y+name",
 209             boxpoints="all",
 210             jitter=0.3,
 211             marker=dict(color=_get_color(idx))
 212         )
 213         tput_traces.append(go.Box(**tput_kwargs))
 214         show_tput = True
 215
 216         if itm["testtype"] == "pdr":
 217             y_lat_row = itm_data[_VALUE["pdr-lat"]].to_list()
 218             y_lat = [y * norm_factor for y in y_lat_row]
 219             if y_lat:
 220                 y_lat_max = max(y_lat) if max(y_lat) > y_lat_max else y_lat_max
 221             nr_of_samples = len(y_lat)
 222             lat_kwargs = dict(
 223                 y=y_lat,
 224                 name=(
 225                     f"{idx + 1}. "
 226                     f"({nr_of_samples:02d} "
 227                     f"run{u's' if nr_of_samples > 1 else u''}) "
 228                     f"{itm['id']}"
 229                 ),
 230                 hoverinfo="all",
 231                 boxpoints="all",
 232                 jitter=0.3,
 233                 marker=dict(color=_get_color(idx))
 234             )
 235             x_lat.append(idx + 1)
 236             lat_traces.append(go.Box(**lat_kwargs))
 237             show_latency = True
 238         else:
 239             lat_traces.append(go.Box())
 240
 241     if show_tput:
 242         pl_tput = deepcopy(layout["plot-throughput"])
 243         pl_tput["xaxis"]["tickvals"] = [i for i in range(len(sel))]
 244         pl_tput["xaxis"]["ticktext"] = [str(i + 1) for i in range(len(sel))]
 245         if y_tput_max:
 246             pl_tput["yaxis"]["range"] = [0, (int(y_tput_max / 1e6) + 1) * 1e6]
 247         fig_tput = go.Figure(data=tput_traces, layout=pl_tput)
 248
 249     if show_latency:
 250         pl_lat = deepcopy(layout["plot-latency"])
 251         pl_lat["xaxis"]["tickvals"] = [i for i in range(len(x_lat))]
 252         pl_lat["xaxis"]["ticktext"] = x_lat
 253         if y_lat_max:
 254             pl_lat["yaxis"]["range"] = [0, (int(y_lat_max / 10) + 1) * 10]
 255         fig_lat = go.Figure(data=lat_traces, layout=pl_lat)
 256
 257     return fig_tput, fig_lat
 258
 259
 260 def table_comparison(data: pd.DataFrame, sel:dict,
 261         normalize: bool) -> pd.DataFrame:
 262     """
 263     """
 264     table = pd.DataFrame(
 265         {
 266             "Test Case": [
 267                 "64b-2t1c-avf-eth-l2xcbase-eth-2memif-1dcr",
 268                 "64b-2t1c-avf-eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc",
 269                 "64b-2t1c-avf-ethip4udp-ip4base-iacl50sl-10kflows",
 270                 "78b-2t1c-avf-ethip6-ip6scale2m-rnd "],
 271             "2106.0-8": [
 272                 "14.45 +- 0.08",
 273                 "9.63 +- 0.05",
 274                 "9.7 +- 0.02",
 275                 "8.95 +- 0.06"],
 276             "2110.0-8": [
 277                 "14.45 +- 0.08",
 278                 "9.63 +- 0.05",
 279                 "9.7 +- 0.02",
 280                 "8.95 +- 0.06"],
 281             "2110.0-9": [
 282                 "14.45 +- 0.08",
 283                 "9.63 +- 0.05",
 284                 "9.7 +- 0.02",
 285                 "8.95 +- 0.06"],
 286             "2202.0-9": [
 287                 "14.45 +- 0.08",
 288                 "9.63 +- 0.05",
 289                 "9.7 +- 0.02",
 290                 "8.95 +- 0.06"],
 291             "2110.0-9 vs 2110.0-8": [
 292                 "-0.23 +-  0.62",
 293                 "-1.37 +-   1.3",
 294                 "+0.08 +-   0.2",
 295                 "-2.16 +-  0.83"],
 296             "2202.0-9 vs 2110.0-9": [
 297                 "+6.95 +-  0.72",
 298                 "+5.35 +-  1.26",
 299                 "+4.48 +-  1.48",
 300                 "+4.09 +-  0.95"]
 301         }
 302     )
 303
 304     return pd.DataFrame()  #table
 305
 306
 307 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 308     """
 309     """
 310
 311     fig = None
 312
 313     traces = list()
 314     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
 315         try:
 316             decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
 317         except (hdrh.codec.HdrLengthException, TypeError) as err:
 318             continue
 319         previous_x = 0.0
 320         prev_perc = 0.0
 321         xaxis = list()
 322         yaxis = list()
 323         hovertext = list()
 324         for item in decoded.get_recorded_iterator():
 325             # The real value is "percentile".
 326             # For 100%, we cut that down to "x_perc" to avoid
 327             # infinity.
 328             percentile = item.percentile_level_iterated_to
 329             x_perc = min(percentile, PERCENTILE_MAX)
 330             xaxis.append(previous_x)
 331             yaxis.append(item.value_iterated_to)
 332             hovertext.append(
 333                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 334                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 335                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 336                 f"Latency: {item.value_iterated_to}uSec"
 337             )
 338             next_x = 100.0 / (100.0 - x_perc)
 339             xaxis.append(next_x)
 340             yaxis.append(item.value_iterated_to)
 341             hovertext.append(
 342                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 343                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 344                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 345                 f"Latency: {item.value_iterated_to}uSec"
 346             )
 347             previous_x = next_x
 348             prev_perc = percentile
 349
 350         traces.append(
 351             go.Scatter(
 352                 x=xaxis,
 353                 y=yaxis,
 354                 name=_GRAPH_LAT_HDRH_DESC[lat_name],
 355                 mode="lines",
 356                 legendgroup=_GRAPH_LAT_HDRH_DESC[lat_name],
 357                 showlegend=bool(idx % 2),
 358                 line=dict(
 359                     color=_get_color(int(idx/2)),
 360                     dash="solid",
 361                     width=1 if idx % 2 else 2
 362                 ),
 363                 hovertext=hovertext,
 364                 hoverinfo="text"
 365             )
 366         )
 367     if traces:
 368         fig = go.Figure()
 369         fig.add_traces(traces)
 370         layout_hdrh = layout.get("plot-hdrh-latency", None)
 371         if lat_hdrh:
 372             fig.update_layout(layout_hdrh)
 373
 374     return fig