resources/tools/dash/app/pal/report/graphs.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """
  15 """
  16
  17 import re
  18 import plotly.graph_objects as go
  19 import pandas as pd
  20
  21 from copy import deepcopy
  22
  23 import hdrh.histogram
  24 import hdrh.codec
  25
  26
  27 _FREQURENCY = {  # [GHz]
  28     "2n-aws": 1.000,
  29     "2n-dnv": 2.000,
  30     "2n-clx": 2.300,
  31     "2n-icx": 2.600,
  32     "2n-skx": 2.500,
  33     "2n-tx2": 2.500,
  34     "2n-zn2": 2.900,
  35     "3n-alt": 3.000,
  36     "3n-aws": 1.000,
  37     "3n-dnv": 2.000,
  38     "3n-icx": 2.600,
  39     "3n-skx": 2.500,
  40     "3n-tsh": 2.200
  41 }
  42
  43 _VALUE = {
  44     "mrr": "result_receive_rate_rate_values",
  45     "ndr": "result_ndr_lower_rate_value",
  46     "pdr": "result_pdr_lower_rate_value",
  47     "pdr-lat": "result_latency_forward_pdr_50_avg"
  48 }
  49 _UNIT = {
  50     "mrr": "result_receive_rate_rate_unit",
  51     "ndr": "result_ndr_lower_rate_unit",
  52     "pdr": "result_pdr_lower_rate_unit",
  53     "pdr-lat": "result_latency_forward_pdr_50_unit"
  54 }
  55 _LAT_HDRH = (  # Do not change the order
  56     "result_latency_forward_pdr_0_hdrh",
  57     "result_latency_reverse_pdr_0_hdrh",
  58     "result_latency_forward_pdr_10_hdrh",
  59     "result_latency_reverse_pdr_10_hdrh",
  60     "result_latency_forward_pdr_50_hdrh",
  61     "result_latency_reverse_pdr_50_hdrh",
  62     "result_latency_forward_pdr_90_hdrh",
  63     "result_latency_reverse_pdr_90_hdrh",
  64 )
  65 # This value depends on latency stream rate (9001 pps) and duration (5s).
  66 # Keep it slightly higher to ensure rounding errors to not remove tick mark.
  67 PERCENTILE_MAX = 99.999501
  68
  69 _GRAPH_LAT_HDRH_DESC = {
  70     "result_latency_forward_pdr_0_hdrh": "No-load.",
  71     "result_latency_reverse_pdr_0_hdrh": "No-load.",
  72     "result_latency_forward_pdr_10_hdrh": "Low-load, 10% PDR.",
  73     "result_latency_reverse_pdr_10_hdrh": "Low-load, 10% PDR.",
  74     "result_latency_forward_pdr_50_hdrh": "Mid-load, 50% PDR.",
  75     "result_latency_reverse_pdr_50_hdrh": "Mid-load, 50% PDR.",
  76     "result_latency_forward_pdr_90_hdrh": "High-load, 90% PDR.",
  77     "result_latency_reverse_pdr_90_hdrh": "High-load, 90% PDR."
  78 }
  79
  80
  81 def _get_color(idx: int) -> str:
  82     """
  83     """
  84     _COLORS = (
  85         "#1A1110", "#DA2647", "#214FC6", "#01786F", "#BD8260", "#FFD12A",
  86         "#A6E7FF", "#738276", "#C95A49", "#FC5A8D", "#CEC8EF", "#391285",
  87         "#6F2DA8", "#FF878D", "#45A27D", "#FFD0B9", "#FD5240", "#DB91EF",
  88         "#44D7A8", "#4F86F7", "#84DE02", "#FFCFF1", "#614051"
  89     )
  90     return _COLORS[idx % len(_COLORS)]
  91
  92
  93 def get_short_version(version: str, dut_type: str="vpp") -> str:
  94     """
  95     """
  96
  97     if dut_type in ("trex", "dpdk"):
  98         return version
  99
 100     s_version = str()
 101     groups = re.search(
 102         pattern=re.compile(r"^(\d{2}).(\d{2})-(rc0|rc1|rc2|release$)"),
 103         string=version
 104     )
 105     if groups:
 106         try:
 107             s_version = \
 108                 f"{groups.group(1)}.{groups.group(2)}.{groups.group(3)}".\
 109                     replace("release", "rls")
 110         except IndexError:
 111             pass
 112
 113     return s_version
 114
 115
 116 def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
 117     """
 118     """
 119
 120     phy = itm["phy"].split("-")
 121     if len(phy) == 4:
 122         topo, arch, nic, drv = phy
 123         if drv == "dpdk":
 124             drv = ""
 125         else:
 126             drv += "-"
 127             drv = drv.replace("_", "-")
 128     else:
 129         return None
 130
 131     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
 132     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
 133     dut_v100 = "none" if itm["dut"] == "trex" else itm["dut"]
 134     dut_v101 = itm["dut"]
 135
 136     df = data.loc[(
 137         (data["release"] == itm["rls"]) &
 138         (
 139             (
 140                 (data["version"] == "1.0.0") &
 141                 (data["dut_type"].str.lower() == dut_v100)
 142             ) |
 143             (
 144                 (data["version"] == "1.0.1") &
 145                 (data["dut_type"].str.lower() == dut_v101)
 146             )
 147         ) &
 148         (data["test_type"] == ttype) &
 149         (data["passed"] == True)
 150     )]
 151     regex_test = \
 152         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$"
 153     df = df[
 154         (df.job.str.endswith(f"{topo}-{arch}")) &
 155         (df.dut_version.str.contains(itm["dutver"].replace(".r", "-r").\
 156             replace("rls", "release"))) &
 157         (df.test_id.str.contains(regex_test, regex=True))
 158     ]
 159
 160     return df
 161
 162
 163 def graph_iterative(data: pd.DataFrame, sel:dict, layout: dict,
 164         normalize: bool) -> tuple:
 165     """
 166     """
 167
 168     fig_tput = None
 169     fig_lat = None
 170
 171     tput_traces = list()
 172     y_tput_max = 0
 173     lat_traces = list()
 174     y_lat_max = 0
 175     x_lat = list()
 176     show_latency = False
 177     show_tput = False
 178     for idx, itm in enumerate(sel):
 179         itm_data = select_iterative_data(data, itm)
 180         if itm_data.empty:
 181             continue
 182         phy = itm["phy"].split("-")
 183         topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 184         norm_factor = 2.0 / _FREQURENCY[topo_arch] if normalize else 1.0
 185         if itm["testtype"] == "mrr":
 186             y_data_raw = itm_data[_VALUE[itm["testtype"]]].to_list()[0]
 187             y_data = [y * norm_factor for y in y_data_raw]
 188             if len(y_data) > 0:
 189                 y_tput_max = \
 190                     max(y_data) if max(y_data) > y_tput_max else y_tput_max
 191         else:
 192             y_data_raw = itm_data[_VALUE[itm["testtype"]]].to_list()
 193             y_data = [y * norm_factor for y in y_data_raw]
 194             if y_data:
 195                 y_tput_max = \
 196                     max(y_data) if max(y_data) > y_tput_max else y_tput_max
 197         nr_of_samples = len(y_data)
 198         tput_kwargs = dict(
 199             y=y_data,
 200             name=(
 201                 f"{idx + 1}. "
 202                 f"({nr_of_samples:02d} "
 203                 f"run{'s' if nr_of_samples > 1 else ''}) "
 204                 f"{itm['id']}"
 205             ),
 206             hoverinfo=u"y+name",
 207             boxpoints="all",
 208             jitter=0.3,
 209             marker=dict(color=_get_color(idx))
 210         )
 211         tput_traces.append(go.Box(**tput_kwargs))
 212         show_tput = True
 213
 214         if itm["testtype"] == "pdr":
 215             y_lat_row = itm_data[_VALUE["pdr-lat"]].to_list()
 216             y_lat = [y * norm_factor for y in y_lat_row]
 217             if y_lat:
 218                 y_lat_max = max(y_lat) if max(y_lat) > y_lat_max else y_lat_max
 219             nr_of_samples = len(y_lat)
 220             lat_kwargs = dict(
 221                 y=y_lat,
 222                 name=(
 223                     f"{idx + 1}. "
 224                     f"({nr_of_samples:02d} "
 225                     f"run{u's' if nr_of_samples > 1 else u''}) "
 226                     f"{itm['id']}"
 227                 ),
 228                 hoverinfo="all",
 229                 boxpoints="all",
 230                 jitter=0.3,
 231                 marker=dict(color=_get_color(idx))
 232             )
 233             x_lat.append(idx + 1)
 234             lat_traces.append(go.Box(**lat_kwargs))
 235             show_latency = True
 236         else:
 237             lat_traces.append(go.Box())
 238
 239     if show_tput:
 240         pl_tput = deepcopy(layout["plot-throughput"])
 241         pl_tput["xaxis"]["tickvals"] = [i for i in range(len(sel))]
 242         pl_tput["xaxis"]["ticktext"] = [str(i + 1) for i in range(len(sel))]
 243         if y_tput_max:
 244             pl_tput["yaxis"]["range"] = [0, (int(y_tput_max / 1e6) + 1) * 1e6]
 245         fig_tput = go.Figure(data=tput_traces, layout=pl_tput)
 246
 247     if show_latency:
 248         pl_lat = deepcopy(layout["plot-latency"])
 249         pl_lat["xaxis"]["tickvals"] = [i for i in range(len(x_lat))]
 250         pl_lat["xaxis"]["ticktext"] = x_lat
 251         if y_lat_max:
 252             pl_lat["yaxis"]["range"] = [0, (int(y_lat_max / 10) + 1) * 10]
 253         fig_lat = go.Figure(data=lat_traces, layout=pl_lat)
 254
 255     return fig_tput, fig_lat
 256
 257
 258 def table_comparison(data: pd.DataFrame, sel:dict,
 259         normalize: bool) -> pd.DataFrame:
 260     """
 261     """
 262     table = pd.DataFrame(
 263         {
 264             "Test Case": [
 265                 "64b-2t1c-avf-eth-l2xcbase-eth-2memif-1dcr",
 266                 "64b-2t1c-avf-eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc",
 267                 "64b-2t1c-avf-ethip4udp-ip4base-iacl50sl-10kflows",
 268                 "78b-2t1c-avf-ethip6-ip6scale2m-rnd "],
 269             "2106.0-8": [
 270                 "14.45 +- 0.08",
 271                 "9.63 +- 0.05",
 272                 "9.7 +- 0.02",
 273                 "8.95 +- 0.06"],
 274             "2110.0-8": [
 275                 "14.45 +- 0.08",
 276                 "9.63 +- 0.05",
 277                 "9.7 +- 0.02",
 278                 "8.95 +- 0.06"],
 279             "2110.0-9": [
 280                 "14.45 +- 0.08",
 281                 "9.63 +- 0.05",
 282                 "9.7 +- 0.02",
 283                 "8.95 +- 0.06"],
 284             "2202.0-9": [
 285                 "14.45 +- 0.08",
 286                 "9.63 +- 0.05",
 287                 "9.7 +- 0.02",
 288                 "8.95 +- 0.06"],
 289             "2110.0-9 vs 2110.0-8": [
 290                 "-0.23 +-  0.62",
 291                 "-1.37 +-   1.3",
 292                 "+0.08 +-   0.2",
 293                 "-2.16 +-  0.83"],
 294             "2202.0-9 vs 2110.0-9": [
 295                 "+6.95 +-  0.72",
 296                 "+5.35 +-  1.26",
 297                 "+4.48 +-  1.48",
 298                 "+4.09 +-  0.95"]
 299         }
 300     )
 301
 302     return pd.DataFrame()  #table
 303
 304
 305 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 306     """
 307     """
 308
 309     fig = None
 310
 311     traces = list()
 312     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
 313         try:
 314             decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
 315         except (hdrh.codec.HdrLengthException, TypeError) as err:
 316             continue
 317         previous_x = 0.0
 318         prev_perc = 0.0
 319         xaxis = list()
 320         yaxis = list()
 321         hovertext = list()
 322         for item in decoded.get_recorded_iterator():
 323             # The real value is "percentile".
 324             # For 100%, we cut that down to "x_perc" to avoid
 325             # infinity.
 326             percentile = item.percentile_level_iterated_to
 327             x_perc = min(percentile, PERCENTILE_MAX)
 328             xaxis.append(previous_x)
 329             yaxis.append(item.value_iterated_to)
 330             hovertext.append(
 331                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 332                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 333                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 334                 f"Latency: {item.value_iterated_to}uSec"
 335             )
 336             next_x = 100.0 / (100.0 - x_perc)
 337             xaxis.append(next_x)
 338             yaxis.append(item.value_iterated_to)
 339             hovertext.append(
 340                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 341                 f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
 342                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 343                 f"Latency: {item.value_iterated_to}uSec"
 344             )
 345             previous_x = next_x
 346             prev_perc = percentile
 347
 348         traces.append(
 349             go.Scatter(
 350                 x=xaxis,
 351                 y=yaxis,
 352                 name=_GRAPH_LAT_HDRH_DESC[lat_name],
 353                 mode="lines",
 354                 legendgroup=_GRAPH_LAT_HDRH_DESC[lat_name],
 355                 showlegend=bool(idx % 2),
 356                 line=dict(
 357                     color=_get_color(int(idx/2)),
 358                     dash="solid",
 359                     width=1 if idx % 2 else 2
 360                 ),
 361                 hovertext=hovertext,
 362                 hoverinfo="text"
 363             )
 364         )
 365     if traces:
 366         fig = go.Figure()
 367         fig.add_traces(traces)
 368         layout_hdrh = layout.get("plot-hdrh-latency", None)
 369         if lat_hdrh:
 370             fig.update_layout(layout_hdrh)
 371
 372     return fig