resources/tools/dash/app/pal/report/graphs.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """
  15 """
  16
  17 import re
  18 import plotly.graph_objects as go
  19 import pandas as pd
  20
  21 from copy import deepcopy
  22
  23 import hdrh.histogram
  24 import hdrh.codec
  25
  26
  27 _VALUE = {
  28     "mrr": "result_receive_rate_rate_values",
  29     "ndr": "result_ndr_lower_rate_value",
  30     "pdr": "result_pdr_lower_rate_value",
  31     "pdr-lat": "result_latency_forward_pdr_50_avg"
  32 }
  33 _UNIT = {
  34     "mrr": "result_receive_rate_rate_unit",
  35     "ndr": "result_ndr_lower_rate_unit",
  36     "pdr": "result_pdr_lower_rate_unit",
  37     "pdr-lat": "result_latency_forward_pdr_50_unit"
  38 }
  39 _LAT_HDRH = (  # Do not change the order
  40     "result_latency_forward_pdr_0_hdrh",
  41     "result_latency_reverse_pdr_0_hdrh",
  42     "result_latency_forward_pdr_10_hdrh",
  43     "result_latency_reverse_pdr_10_hdrh",
  44     "result_latency_forward_pdr_50_hdrh",
  45     "result_latency_reverse_pdr_50_hdrh",
  46     "result_latency_forward_pdr_90_hdrh",
  47     "result_latency_reverse_pdr_90_hdrh",
  48 )
  49 # This value depends on latency stream rate (9001 pps) and duration (5s).
  50 # Keep it slightly higher to ensure rounding errors to not remove tick mark.
  51 PERCENTILE_MAX = 99.999501
  52
  53 _GRAPH_LAT_HDRH_DESC = {
  54     u"result_latency_forward_pdr_0_hdrh": u"No-load.",
  55     u"result_latency_reverse_pdr_0_hdrh": u"No-load.",
  56     u"result_latency_forward_pdr_10_hdrh": u"Low-load, 10% PDR.",
  57     u"result_latency_reverse_pdr_10_hdrh": u"Low-load, 10% PDR.",
  58     u"result_latency_forward_pdr_50_hdrh": u"Mid-load, 50% PDR.",
  59     u"result_latency_reverse_pdr_50_hdrh": u"Mid-load, 50% PDR.",
  60     u"result_latency_forward_pdr_90_hdrh": u"High-load, 90% PDR.",
  61     u"result_latency_reverse_pdr_90_hdrh": u"High-load, 90% PDR."
  62 }
  63 REG_EX_VPP_VERSION = re.compile(r"^(\d{2}).(\d{2})-(rc0|rc1|rc2|release$)")
  64
  65
  66 def _get_color(idx: int) -> str:
  67     """
  68     """
  69     _COLORS = (
  70         "#1A1110", "#DA2647", "#214FC6", "#01786F", "#BD8260", "#FFD12A",
  71         "#A6E7FF", "#738276", "#C95A49", "#FC5A8D", "#CEC8EF", "#391285",
  72         "#6F2DA8", "#FF878D", "#45A27D", "#FFD0B9", "#FD5240", "#DB91EF",
  73         "#44D7A8", "#4F86F7", "#84DE02", "#FFCFF1", "#614051"
  74     )
  75     return _COLORS[idx % len(_COLORS)]
  76
  77
  78 def get_short_version(version: str, dut_type: str="vpp") -> str:
  79     """
  80     """
  81
  82     if dut_type in ("trex", "dpdk"):
  83         return version
  84
  85     s_version = str()
  86     groups = re.search(pattern=REG_EX_VPP_VERSION, string=version)
  87     if groups:
  88         try:
  89             s_version = f"{groups.group(1)}.{groups.group(2)}_{groups.group(3)}"
  90         except IndexError:
  91             pass
  92
  93     return s_version
  94
  95
  96 def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
  97     """
  98     """
  99
 100     phy = itm["phy"].split("-")
 101     if len(phy) == 4:
 102         topo, arch, nic, drv = phy
 103         if drv == "dpdk":
 104             drv = ""
 105         else:
 106             drv += "-"
 107             drv = drv.replace("_", "-")
 108     else:
 109         return None
 110
 111     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
 112     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
 113     dut_v100 = "none" if itm["dut"] == "trex" else itm["dut"]
 114     dut_v101 = itm["dut"]
 115
 116     df = data.loc[(
 117         (data["release"] == itm["rls"]) &
 118         (
 119             (
 120                 (data["version"] == "1.0.0") &
 121                 (data["dut_type"].str.lower() == dut_v100)
 122             ) |
 123             (
 124                 (data["version"] == "1.0.1") &
 125                 (data["dut_type"].str.lower() == dut_v101)
 126             )
 127         ) &
 128         (data["test_type"] == ttype) &
 129         (data["passed"] == True)
 130     )]
 131     regex_test = \
 132         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$"
 133     df = df[
 134         (df.job.str.endswith(f"{topo}-{arch}")) &
 135         (df.dut_version.str.contains(itm["dutver"].replace("_", "-"))) &
 136         (df.test_id.str.contains(regex_test, regex=True))
 137     ]
 138
 139     return df
 140
 141
 142 def graph_iterative(data: pd.DataFrame, sel:dict, layout: dict) -> tuple:
 143     """
 144     """
 145
 146     fig_tput = None
 147     fig_lat = None
 148
 149     tput_traces = list()
 150     y_tput_max = 0
 151     lat_traces = list()
 152     y_lat_max = 0
 153     x_lat = list()
 154     for idx, itm in enumerate(sel):
 155         itm_data = select_iterative_data(data, itm)
 156         if itm["testtype"] == "mrr":
 157             y_data = itm_data[_VALUE[itm["testtype"]]].to_list()[0]
 158             if y_data.size > 0:
 159                 y_tput_max = \
 160                     max(y_data) if max(y_data) > y_tput_max else y_tput_max
 161         else:
 162             y_data = itm_data[_VALUE[itm["testtype"]]].to_list()
 163             if y_data:
 164                 y_tput_max = \
 165                     max(y_data) if max(y_data) > y_tput_max else y_tput_max
 166         nr_of_samples = len(y_data)
 167         tput_kwargs = dict(
 168             y=y_data,
 169             name=(
 170                 f"{idx + 1}. "
 171                 f"({nr_of_samples:02d} "
 172                 f"run{u's' if nr_of_samples > 1 else u''}) "
 173                 f"{itm['id']}"
 174             ),
 175             hoverinfo=u"y+name",
 176             boxpoints="all",
 177             jitter=0.3,
 178             marker=dict(color=_get_color(idx))
 179         )
 180         tput_traces.append(go.Box(**tput_kwargs))
 181
 182         show_latency = False
 183         if itm["testtype"] == "pdr":
 184             y_lat = itm_data[_VALUE["pdr-lat"]].to_list()
 185             if y_lat:
 186                 y_lat_max = max(y_lat) if max(y_lat) > y_lat_max else y_lat_max
 187             nr_of_samples = len(y_lat)
 188             lat_kwargs = dict(
 189                 y=y_lat,
 190                 name=(
 191                     f"{idx + 1}. "
 192                     f"({nr_of_samples:02d} "
 193                     f"run{u's' if nr_of_samples > 1 else u''}) "
 194                     f"{itm['id']}"
 195                 ),
 196                 hoverinfo=u"y+name",
 197                 boxpoints="all",
 198                 jitter=0.3,
 199                 marker=dict(color=_get_color(idx))
 200             )
 201             x_lat.append(idx + 1)
 202             lat_traces.append(go.Box(**lat_kwargs))
 203             show_latency = True
 204         else:
 205             lat_traces.append(go.Box())
 206
 207     pl_tput = deepcopy(layout["plot-throughput"])
 208     pl_tput[u"xaxis"][u"tickvals"] = [i for i in range(len(sel))]
 209     pl_tput[u"xaxis"][u"ticktext"] = [str(i + 1) for i in range(len(sel))]
 210     if y_tput_max:
 211         pl_tput[u"yaxis"][u"range"] = [0, (int(y_tput_max / 1e6) + 1) * 1e6]
 212     fig_tput = go.Figure(data=tput_traces, layout=pl_tput)
 213
 214     if show_latency:
 215         pl_lat = deepcopy(layout["plot-latency"])
 216         pl_lat[u"xaxis"][u"tickvals"] = [i for i in range(len(x_lat))]
 217         pl_lat[u"xaxis"][u"ticktext"] = x_lat
 218         if y_lat_max:
 219             pl_lat[u"yaxis"][u"range"] = [0, (int(y_lat_max / 10) + 1) * 10]
 220         fig_lat = go.Figure(data=lat_traces, layout=pl_lat)
 221
 222     return fig_tput, fig_lat
 223
 224
 225 def table_comparison(data: pd.DataFrame, sel:dict) -> pd.DataFrame:
 226     """
 227     """
 228     table = pd.DataFrame(
 229         {
 230             "Test Case": [
 231                 "64b-2t1c-avf-eth-l2xcbase-eth-2memif-1dcr",
 232                 "64b-2t1c-avf-eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc",
 233                 "64b-2t1c-avf-ethip4udp-ip4base-iacl50sl-10kflows",
 234                 "78b-2t1c-avf-ethip6-ip6scale2m-rnd "],
 235             "2106.0-8": [
 236                 "14.45 +- 0.08",
 237                 "9.63 +- 0.05",
 238                 "9.7 +- 0.02",
 239                 "8.95 +- 0.06"],
 240             "2110.0-8": [
 241                 "14.45 +- 0.08",
 242                 "9.63 +- 0.05",
 243                 "9.7 +- 0.02",
 244                 "8.95 +- 0.06"],
 245             "2110.0-9": [
 246                 "14.45 +- 0.08",
 247                 "9.63 +- 0.05",
 248                 "9.7 +- 0.02",
 249                 "8.95 +- 0.06"],
 250             "2202.0-9": [
 251                 "14.45 +- 0.08",
 252                 "9.63 +- 0.05",
 253                 "9.7 +- 0.02",
 254                 "8.95 +- 0.06"],
 255             "2110.0-9 vs 2110.0-8": [
 256                 "-0.23 +-  0.62",
 257                 "-1.37 +-   1.3",
 258                 "+0.08 +-   0.2",
 259                 "-2.16 +-  0.83"],
 260             "2202.0-9 vs 2110.0-9": [
 261                 "+6.95 +-  0.72",
 262                 "+5.35 +-  1.26",
 263                 "+4.48 +-  1.48",
 264                 "+4.09 +-  0.95"]
 265         }
 266     )
 267
 268     return pd.DataFrame()  #table
 269
 270
 271 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
 272     """
 273     """
 274
 275     fig = None
 276
 277     traces = list()
 278     for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
 279         try:
 280             decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
 281         except (hdrh.codec.HdrLengthException, TypeError) as err:
 282             continue
 283         previous_x = 0.0
 284         prev_perc = 0.0
 285         xaxis = list()
 286         yaxis = list()
 287         hovertext = list()
 288         for item in decoded.get_recorded_iterator():
 289             # The real value is "percentile".
 290             # For 100%, we cut that down to "x_perc" to avoid
 291             # infinity.
 292             percentile = item.percentile_level_iterated_to
 293             x_perc = min(percentile, PERCENTILE_MAX)
 294             xaxis.append(previous_x)
 295             yaxis.append(item.value_iterated_to)
 296             hovertext.append(
 297                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 298                 f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
 299                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 300                 f"Latency: {item.value_iterated_to}uSec"
 301             )
 302             next_x = 100.0 / (100.0 - x_perc)
 303             xaxis.append(next_x)
 304             yaxis.append(item.value_iterated_to)
 305             hovertext.append(
 306                 f"<b>{_GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
 307                 f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
 308                 f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 309                 f"Latency: {item.value_iterated_to}uSec"
 310             )
 311             previous_x = next_x
 312             prev_perc = percentile
 313
 314         traces.append(
 315             go.Scatter(
 316                 x=xaxis,
 317                 y=yaxis,
 318                 name=_GRAPH_LAT_HDRH_DESC[lat_name],
 319                 mode=u"lines",
 320                 legendgroup=_GRAPH_LAT_HDRH_DESC[lat_name],
 321                 showlegend=bool(idx % 2),
 322                 line=dict(
 323                     color=_get_color(int(idx/2)),
 324                     dash=u"solid",
 325                     width=1 if idx % 2 else 2
 326                 ),
 327                 hovertext=hovertext,
 328                 hoverinfo=u"text"
 329             )
 330         )
 331     if traces:
 332         fig = go.Figure()
 333         fig.add_traces(traces)
 334         layout_hdrh = layout.get("plot-hdrh-latency", None)
 335         if lat_hdrh:
 336             fig.update_layout(layout_hdrh)
 337
 338     return fig