resources/tools/presentation/generator_plots.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate plots.
  15 """
  16
  17
  18 import re
  19 import logging
  20
  21 from collections import OrderedDict
  22 from datetime import datetime
  23 from copy import deepcopy
  24 from math import log
  25
  26 import hdrh.histogram
  27 import hdrh.codec
  28 import pandas as pd
  29 import plotly.offline as ploff
  30 import plotly.graph_objs as plgo
  31 import plotly.exceptions as plerr
  32
  33 from plotly.exceptions import PlotlyError
  34
  35 from pal_utils import mean, stdev
  36
  37
  38 COLORS = (
  39     u"#1A1110",
  40     u"#DA2647",
  41     u"#214FC6",
  42     u"#01786F",
  43     u"#BD8260",
  44     u"#FFD12A",
  45     u"#A6E7FF",
  46     u"#738276",
  47     u"#C95A49",
  48     u"#FC5A8D",
  49     u"#CEC8EF",
  50     u"#391285",
  51     u"#6F2DA8",
  52     u"#FF878D",
  53     u"#45A27D",
  54     u"#FFD0B9",
  55     u"#FD5240",
  56     u"#DB91EF",
  57     u"#44D7A8",
  58     u"#4F86F7",
  59     u"#84DE02",
  60     u"#FFCFF1",
  61     u"#614051"
  62 )
  63
  64 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)-')
  65
  66 # This value depends on latency stream rate (9001 pps) and duration (5s).
  67 # Keep it slightly higher to ensure rounding errors to not remove tick mark.
  68 PERCENTILE_MAX = 99.999501
  69
  70
  71 def generate_plots(spec, data):
  72     """Generate all plots specified in the specification file.
  73
  74     :param spec: Specification read from the specification file.
  75     :param data: Data to process.
  76     :type spec: Specification
  77     :type data: InputData
  78     """
  79
  80     generator = {
  81         u"plot_nf_reconf_box_name": plot_nf_reconf_box_name,
  82         u"plot_perf_box_name": plot_perf_box_name,
  83         u"plot_tsa_name": plot_tsa_name,
  84         u"plot_http_server_perf_box": plot_http_server_perf_box,
  85         u"plot_nf_heatmap": plot_nf_heatmap,
  86         u"plot_hdrh_lat_by_percentile": plot_hdrh_lat_by_percentile,
  87         u"plot_hdrh_lat_by_percentile_x_log": plot_hdrh_lat_by_percentile_x_log,
  88         u"plot_mrr_box_name": plot_mrr_box_name,
  89         u"plot_ndrpdr_box_name": plot_ndrpdr_box_name,
  90         u"plot_statistics": plot_statistics
  91     }
  92
  93     logging.info(u"Generating the plots ...")
  94     for index, plot in enumerate(spec.plots):
  95         try:
  96             logging.info(f"  Plot nr {index + 1}: {plot.get(u'title', u'')}")
  97             plot[u"limits"] = spec.environment[u"limits"]
  98             generator[plot[u"algorithm"]](plot, data)
  99             logging.info(u"  Done.")
 100         except NameError as err:
 101             logging.error(
 102                 f"Probably algorithm {plot[u'algorithm']} is not defined: "
 103                 f"{repr(err)}"
 104             )
 105     logging.info(u"Done.")
 106
 107
 108 def plot_statistics(plot, input_data):
 109     """Generate the plot(s) with algorithm: plot_statistics
 110     specified in the specification file.
 111
 112     :param plot: Plot to generate.
 113     :param input_data: Data to process.
 114     :type plot: pandas.Series
 115     :type input_data: InputData
 116     """
 117
 118     data_x = list()
 119     data_y_pass = list()
 120     data_y_fail = list()
 121     data_y_duration = list()
 122     hover_text = list()
 123     hover_str = (
 124         u"date: {date}<br>"
 125         u"passed: {passed}<br>"
 126         u"failed: {failed}<br>"
 127         u"duration: {duration}<br>"
 128         u"{sut}-ref: {build}<br>"
 129         u"csit-ref: {test}-{period}-build-{build_nr}<br>"
 130         u"testbed: {testbed}"
 131     )
 132     for job, builds in plot[u"data"].items():
 133         for build_nr in builds:
 134             try:
 135                 meta = input_data.metadata(job, str(build_nr))
 136                 generated = meta[u"generated"]
 137                 date = datetime(
 138                     int(generated[0:4]),
 139                     int(generated[4:6]),
 140                     int(generated[6:8]),
 141                     int(generated[9:11]),
 142                     int(generated[12:])
 143                 )
 144                 d_y_pass = meta[u"tests_passed"]
 145                 d_y_fail = meta[u"tests_failed"]
 146                 minutes = meta[u"elapsedtime"] // 60000
 147                 duration = f"{(minutes // 60):02d}:{(minutes % 60):02d}"
 148                 version = meta[u"version"]
 149             except (KeyError, IndexError, ValueError, AttributeError):
 150                 continue
 151             data_x.append(date)
 152             data_y_pass.append(d_y_pass)
 153             data_y_fail.append(d_y_fail)
 154             data_y_duration.append(minutes)
 155             hover_text.append(hover_str.format(
 156                 date=date,
 157                 passed=d_y_pass,
 158                 failed=d_y_fail,
 159                 duration=duration,
 160                 sut=u"vpp" if u"vpp" in job else u"dpdk",
 161                 build=version,
 162                 test=u"mrr" if u"mrr" in job else u"ndrpdr",
 163                 period=u"daily" if u"daily" in job else u"weekly",
 164                 build_nr=build_nr,
 165                 testbed=meta[u"testbed"]
 166             ))
 167
 168     traces = [
 169         plgo.Bar(
 170             x=data_x,
 171             y=data_y_pass,
 172             name=u"Passed",
 173             text=hover_text,
 174             hoverinfo=u"text"
 175         ),
 176         plgo.Bar(
 177             x=data_x,
 178             y=data_y_fail,
 179             name=u"Failed",
 180             text=hover_text,
 181             hoverinfo=u"text"),
 182         plgo.Scatter(
 183             x=data_x,
 184             y=data_y_duration,
 185             name=u"Duration",
 186             yaxis=u"y2",
 187             text=hover_text,
 188             hoverinfo=u"text"
 189         )
 190     ]
 191
 192     name_file = f"{plot[u'output-file']}.html"
 193
 194     logging.info(f"    Writing the file {name_file}")
 195     plpl = plgo.Figure(data=traces, layout=plot[u"layout"])
 196     tickvals = [0, (max(data_y_duration) // 60) * 60]
 197     step = tickvals[1] / 5
 198     for i in range(5):
 199         tickvals.append(int(tickvals[0] + step * (i + 1)))
 200     plpl.update_layout(
 201         yaxis2=dict(
 202             title=u"Duration [hh:mm]",
 203             anchor=u"x",
 204             overlaying=u"y",
 205             side=u"right",
 206             rangemode="tozero",
 207             tickmode=u"array",
 208             tickvals=tickvals,
 209             ticktext=[f"{(val // 60):02d}:{(val % 60):02d}" for val in tickvals]
 210         )
 211     )
 212     plpl.update_layout(barmode=u"stack")
 213     try:
 214         ploff.plot(
 215             plpl,
 216             show_link=False,
 217             auto_open=False,
 218             filename=name_file
 219         )
 220     except plerr.PlotlyEmptyDataError:
 221         logging.warning(u"No data for the plot. Skipped.")
 222
 223
 224 def plot_hdrh_lat_by_percentile(plot, input_data):
 225     """Generate the plot(s) with algorithm: plot_hdrh_lat_by_percentile
 226     specified in the specification file.
 227
 228     :param plot: Plot to generate.
 229     :param input_data: Data to process.
 230     :type plot: pandas.Series
 231     :type input_data: InputData
 232     """
 233
 234     # Transform the data
 235     logging.info(
 236         f"    Creating the data set for the {plot.get(u'type', u'')} "
 237         f"{plot.get(u'title', u'')}."
 238     )
 239     if plot.get(u"include", None):
 240         data = input_data.filter_tests_by_name(
 241             plot,
 242             params=[u"name", u"latency", u"parent", u"tags", u"type"]
 243         )[0][0]
 244     elif plot.get(u"filter", None):
 245         data = input_data.filter_data(
 246             plot,
 247             params=[u"name", u"latency", u"parent", u"tags", u"type"],
 248             continue_on_error=True
 249         )[0][0]
 250     else:
 251         job = list(plot[u"data"].keys())[0]
 252         build = str(plot[u"data"][job][0])
 253         data = input_data.tests(job, build)
 254
 255     if data is None or len(data) == 0:
 256         logging.error(u"No data.")
 257         return
 258
 259     desc = {
 260         u"LAT0": u"No-load.",
 261         u"PDR10": u"Low-load, 10% PDR.",
 262         u"PDR50": u"Mid-load, 50% PDR.",
 263         u"PDR90": u"High-load, 90% PDR.",
 264         u"PDR": u"Full-load, 100% PDR.",
 265         u"NDR10": u"Low-load, 10% NDR.",
 266         u"NDR50": u"Mid-load, 50% NDR.",
 267         u"NDR90": u"High-load, 90% NDR.",
 268         u"NDR": u"Full-load, 100% NDR."
 269     }
 270
 271     graphs = [
 272         u"LAT0",
 273         u"PDR10",
 274         u"PDR50",
 275         u"PDR90"
 276     ]
 277
 278     file_links = plot.get(u"output-file-links", None)
 279     target_links = plot.get(u"target-links", None)
 280
 281     for test in data:
 282         try:
 283             if test[u"type"] not in (u"NDRPDR",):
 284                 logging.warning(f"Invalid test type: {test[u'type']}")
 285                 continue
 286             name = re.sub(REGEX_NIC, u"", test[u"parent"].
 287                           replace(u'-ndrpdr', u'').replace(u'2n1l-', u''))
 288             try:
 289                 nic = re.search(REGEX_NIC, test[u"parent"]).group(1)
 290             except (IndexError, AttributeError, KeyError, ValueError):
 291                 nic = u""
 292             name_link = f"{nic}-{test[u'name']}".replace(u'-ndrpdr', u'')
 293
 294             logging.info(f"    Generating the graph: {name_link}")
 295
 296             fig = plgo.Figure()
 297             layout = deepcopy(plot[u"layout"])
 298
 299             for color, graph in enumerate(graphs):
 300                 for idx, direction in enumerate((u"direction1", u"direction2")):
 301                     previous_x = 0.0
 302                     xaxis = list()
 303                     yaxis = list()
 304                     hovertext = list()
 305                     try:
 306                         decoded = hdrh.histogram.HdrHistogram.decode(
 307                             test[u"latency"][graph][direction][u"hdrh"]
 308                         )
 309                     except hdrh.codec.HdrLengthException:
 310                         logging.warning(
 311                             f"No data for direction {(u'W-E', u'E-W')[idx % 2]}"
 312                         )
 313                         continue
 314
 315                     for item in decoded.get_recorded_iterator():
 316                         percentile = item.percentile_level_iterated_to
 317                         xaxis.append(previous_x)
 318                         yaxis.append(item.value_iterated_to)
 319                         hovertext.append(
 320                             f"<b>{desc[graph]}</b><br>"
 321                             f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
 322                             f"Percentile: "
 323                             f"{previous_x:.5f}-{percentile:.5f}%<br>"
 324                             f"Latency: {item.value_iterated_to}uSec"
 325                         )
 326                         xaxis.append(percentile)
 327                         yaxis.append(item.value_iterated_to)
 328                         hovertext.append(
 329                             f"<b>{desc[graph]}</b><br>"
 330                             f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
 331                             f"Percentile: "
 332                             f"{previous_x:.5f}-{percentile:.5f}%<br>"
 333                             f"Latency: {item.value_iterated_to}uSec"
 334                         )
 335                         previous_x = percentile
 336                     fig.add_trace(
 337                         plgo.Scatter(
 338                             x=xaxis,
 339                             y=yaxis,
 340                             name=desc[graph],
 341                             mode=u"lines",
 342                             legendgroup=desc[graph],
 343                             showlegend=bool(idx),
 344                             line=dict(
 345                                 color=COLORS[color],
 346                                 dash=u"solid",
 347                                 width=1 if idx % 2 else 2
 348                             ),
 349                             hovertext=hovertext,
 350                             hoverinfo=u"text"
 351                         )
 352                     )
 353
 354             layout[u"title"][u"text"] = f"<b>Latency:</b> {name}"
 355             fig.update_layout(layout)
 356
 357             # Create plot
 358             file_name = f"{plot[u'output-file']}-{name_link}.html"
 359             logging.info(f"    Writing file {file_name}")
 360
 361             try:
 362                 # Export Plot
 363                 ploff.plot(fig, show_link=False, auto_open=False,
 364                            filename=file_name)
 365                 # Add link to the file:
 366                 if file_links and target_links:
 367                     with open(file_links, u"a") as file_handler:
 368                         file_handler.write(
 369                             f"- `{name_link} "
 370                             f"<{target_links}/{file_name.split(u'/')[-1]}>`_\n"
 371                         )
 372             except FileNotFoundError as err:
 373                 logging.error(
 374                     f"Not possible to write the link to the file "
 375                     f"{file_links}\n{err}"
 376                 )
 377             except PlotlyError as err:
 378                 logging.error(f"   Finished with error: {repr(err)}")
 379
 380         except hdrh.codec.HdrLengthException as err:
 381             logging.warning(repr(err))
 382             continue
 383
 384         except (ValueError, KeyError) as err:
 385             logging.warning(repr(err))
 386             continue
 387
 388
 389 def plot_hdrh_lat_by_percentile_x_log(plot, input_data):
 390     """Generate the plot(s) with algorithm: plot_hdrh_lat_by_percentile_x_log
 391     specified in the specification file.
 392
 393     :param plot: Plot to generate.
 394     :param input_data: Data to process.
 395     :type plot: pandas.Series
 396     :type input_data: InputData
 397     """
 398
 399     # Transform the data
 400     logging.info(
 401         f"    Creating the data set for the {plot.get(u'type', u'')} "
 402         f"{plot.get(u'title', u'')}."
 403     )
 404     if plot.get(u"include", None):
 405         data = input_data.filter_tests_by_name(
 406             plot,
 407             params=[u"name", u"latency", u"parent", u"tags", u"type"]
 408         )[0][0]
 409     elif plot.get(u"filter", None):
 410         data = input_data.filter_data(
 411             plot,
 412             params=[u"name", u"latency", u"parent", u"tags", u"type"],
 413             continue_on_error=True
 414         )[0][0]
 415     else:
 416         job = list(plot[u"data"].keys())[0]
 417         build = str(plot[u"data"][job][0])
 418         data = input_data.tests(job, build)
 419
 420     if data is None or len(data) == 0:
 421         logging.error(u"No data.")
 422         return
 423
 424     desc = {
 425         u"LAT0": u"No-load.",
 426         u"PDR10": u"Low-load, 10% PDR.",
 427         u"PDR50": u"Mid-load, 50% PDR.",
 428         u"PDR90": u"High-load, 90% PDR.",
 429         u"PDR": u"Full-load, 100% PDR.",
 430         u"NDR10": u"Low-load, 10% NDR.",
 431         u"NDR50": u"Mid-load, 50% NDR.",
 432         u"NDR90": u"High-load, 90% NDR.",
 433         u"NDR": u"Full-load, 100% NDR."
 434     }
 435
 436     graphs = [
 437         u"LAT0",
 438         u"PDR10",
 439         u"PDR50",
 440         u"PDR90"
 441     ]
 442
 443     file_links = plot.get(u"output-file-links", None)
 444     target_links = plot.get(u"target-links", None)
 445
 446     for test in data:
 447         try:
 448             if test[u"type"] not in (u"NDRPDR",):
 449                 logging.warning(f"Invalid test type: {test[u'type']}")
 450                 continue
 451             name = re.sub(REGEX_NIC, u"", test[u"parent"].
 452                           replace(u'-ndrpdr', u'').replace(u'2n1l-', u''))
 453             try:
 454                 nic = re.search(REGEX_NIC, test[u"parent"]).group(1)
 455             except (IndexError, AttributeError, KeyError, ValueError):
 456                 nic = u""
 457             name_link = f"{nic}-{test[u'name']}".replace(u'-ndrpdr', u'')
 458
 459             logging.info(f"    Generating the graph: {name_link}")
 460
 461             fig = plgo.Figure()
 462             layout = deepcopy(plot[u"layout"])
 463
 464             for color, graph in enumerate(graphs):
 465                 for idx, direction in enumerate((u"direction1", u"direction2")):
 466                     previous_x = 0.0
 467                     prev_perc = 0.0
 468                     xaxis = list()
 469                     yaxis = list()
 470                     hovertext = list()
 471                     try:
 472                         decoded = hdrh.histogram.HdrHistogram.decode(
 473                             test[u"latency"][graph][direction][u"hdrh"]
 474                         )
 475                     except (hdrh.codec.HdrLengthException, TypeError):
 476                         logging.warning(
 477                             f"No data for direction {(u'W-E', u'E-W')[idx % 2]}"
 478                         )
 479                         continue
 480
 481                     for item in decoded.get_recorded_iterator():
 482                         # The real value is "percentile".
 483                         # For 100%, we cut that down to "x_perc" to avoid
 484                         # infinity.
 485                         percentile = item.percentile_level_iterated_to
 486                         x_perc = min(percentile, PERCENTILE_MAX)
 487                         xaxis.append(previous_x)
 488                         yaxis.append(item.value_iterated_to)
 489                         hovertext.append(
 490                             f"<b>{desc[graph]}</b><br>"
 491                             f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
 492                             f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 493                             f"Latency: {item.value_iterated_to}uSec"
 494                         )
 495                         next_x = 100.0 / (100.0 - x_perc)
 496                         xaxis.append(next_x)
 497                         yaxis.append(item.value_iterated_to)
 498                         hovertext.append(
 499                             f"<b>{desc[graph]}</b><br>"
 500                             f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>"
 501                             f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
 502                             f"Latency: {item.value_iterated_to}uSec"
 503                         )
 504                         previous_x = next_x
 505                         prev_perc = percentile
 506                     fig.add_trace(
 507                         plgo.Scatter(
 508                             x=xaxis,
 509                             y=yaxis,
 510                             name=desc[graph],
 511                             mode=u"lines",
 512                             legendgroup=desc[graph],
 513                             showlegend=not(bool(idx)),
 514                             line=dict(
 515                                 color=COLORS[color],
 516                                 dash=u"solid",
 517                                 width=1 if idx % 2 else 2
 518                             ),
 519                             hovertext=hovertext,
 520                             hoverinfo=u"text"
 521                         )
 522                     )
 523
 524             layout[u"title"][u"text"] = f"<b>Latency:</b> {name}"
 525             x_max = log(100.0 / (100.0 - PERCENTILE_MAX), 10)
 526             layout[u"xaxis"][u"range"] = [0, x_max]
 527             fig.update_layout(layout)
 528
 529             # Create plot
 530             file_name = f"{plot[u'output-file']}-{name_link}.html"
 531             logging.info(f"    Writing file {file_name}")
 532
 533             try:
 534                 # Export Plot
 535                 ploff.plot(fig, show_link=False, auto_open=False,
 536                            filename=file_name)
 537                 # Add link to the file:
 538                 if file_links and target_links:
 539                     with open(file_links, u"a") as file_handler:
 540                         file_handler.write(
 541                             f"- `{name_link} "
 542                             f"<{target_links}/{file_name.split(u'/')[-1]}>`_\n"
 543                         )
 544             except FileNotFoundError as err:
 545                 logging.error(
 546                     f"Not possible to write the link to the file "
 547                     f"{file_links}\n{err}"
 548                 )
 549             except PlotlyError as err:
 550                 logging.error(f"   Finished with error: {repr(err)}")
 551
 552         except hdrh.codec.HdrLengthException as err:
 553             logging.warning(repr(err))
 554             continue
 555
 556         except (ValueError, KeyError) as err:
 557             logging.warning(repr(err))
 558             continue
 559
 560
 561 def plot_nf_reconf_box_name(plot, input_data):
 562     """Generate the plot(s) with algorithm: plot_nf_reconf_box_name
 563     specified in the specification file.
 564
 565     :param plot: Plot to generate.
 566     :param input_data: Data to process.
 567     :type plot: pandas.Series
 568     :type input_data: InputData
 569     """
 570
 571     # Transform the data
 572     logging.info(
 573         f"    Creating the data set for the {plot.get(u'type', u'')} "
 574         f"{plot.get(u'title', u'')}."
 575     )
 576     data = input_data.filter_tests_by_name(
 577         plot, params=[u"result", u"parent", u"tags", u"type"]
 578     )
 579     if data is None:
 580         logging.error(u"No data.")
 581         return
 582
 583     for core in plot.get(u"core", tuple()):
 584         # Prepare the data for the plot
 585         y_vals = OrderedDict()
 586         loss = dict()
 587         for item in plot.get(u"include", tuple()):
 588             reg_ex = re.compile(str(item.format(core=core)).lower())
 589             for job in data:
 590                 for build in job:
 591                     for test_id, test in build.iteritems():
 592                         if not re.match(reg_ex, str(test_id).lower()):
 593                             continue
 594                         if y_vals.get(test[u"parent"], None) is None:
 595                             y_vals[test[u"parent"]] = list()
 596                             loss[test[u"parent"]] = list()
 597                         try:
 598                             y_vals[test[u"parent"]].append(
 599                                 test[u"result"][u"time"]
 600                             )
 601                             loss[test[u"parent"]].append(
 602                                 test[u"result"][u"loss"]
 603                             )
 604                         except (KeyError, TypeError):
 605                             y_vals[test[u"parent"]].append(None)
 606
 607         # Add None to the lists with missing data
 608         max_len = 0
 609         nr_of_samples = list()
 610         for val in y_vals.values():
 611             if len(val) > max_len:
 612                 max_len = len(val)
 613             nr_of_samples.append(len(val))
 614         for val in y_vals.values():
 615             if len(val) < max_len:
 616                 val.extend([None for _ in range(max_len - len(val))])
 617
 618         # Add plot traces
 619         traces = list()
 620         df_y = pd.DataFrame(y_vals)
 621         df_y.head()
 622         for i, col in enumerate(df_y.columns):
 623             tst_name = re.sub(
 624                 REGEX_NIC, u"",
 625                 col.lower().replace(u'-reconf', u'').replace(u'2n1l-', u'').
 626                 replace(u'2n-', u'').replace(u'-testpmd', u'')
 627             )
 628             traces.append(plgo.Box(
 629                 x=[str(i + 1) + u'.'] * len(df_y[col]),
 630                 y=df_y[col],
 631                 name=(
 632                     f"{i + 1}. "
 633                     f"({nr_of_samples[i]:02d} "
 634                     f"run{u's' if nr_of_samples[i] > 1 else u''}, "
 635                     f"packets lost average: {mean(loss[col]):.1f}) "
 636                     f"{u'-'.join(tst_name.split(u'-')[2:])}"
 637                 ),
 638                 hoverinfo=u"y+name"
 639             ))
 640         try:
 641             # Create plot
 642             layout = deepcopy(plot[u"layout"])
 643             layout[u"title"] = f"<b>Time Lost:</b> {layout[u'title']}"
 644             layout[u"yaxis"][u"title"] = u"<b>Effective Blocked Time [s]</b>"
 645             layout[u"legend"][u"font"][u"size"] = 14
 646             layout[u"yaxis"].pop(u"range")
 647             plpl = plgo.Figure(data=traces, layout=layout)
 648
 649             # Export Plot
 650             file_name = f"{plot[u'output-file'].format(core=core)}.html"
 651             logging.info(f"    Writing file {file_name}")
 652             ploff.plot(
 653                 plpl,
 654                 show_link=False,
 655                 auto_open=False,
 656                 filename=file_name
 657             )
 658         except PlotlyError as err:
 659             logging.error(
 660                 f"   Finished with error: {repr(err)}".replace(u"\n", u" ")
 661             )
 662
 663
 664 def plot_perf_box_name(plot, input_data):
 665     """Generate the plot(s) with algorithm: plot_perf_box_name
 666     specified in the specification file.
 667
 668     Use only for soak and hoststack tests.
 669
 670     :param plot: Plot to generate.
 671     :param input_data: Data to process.
 672     :type plot: pandas.Series
 673     :type input_data: InputData
 674     """
 675
 676     # Transform the data
 677     logging.info(
 678         f"    Creating data set for the {plot.get(u'type', u'')} "
 679         f"{plot.get(u'title', u'')}."
 680     )
 681     data = input_data.filter_tests_by_name(
 682         plot,
 683         params=[u"throughput", u"gbps", u"result", u"parent", u"tags", u"type"])
 684     if data is None:
 685         logging.error(u"No data.")
 686         return
 687
 688     # Prepare the data for the plot
 689     y_vals = OrderedDict()
 690     test_type = u""
 691
 692     for item in plot.get(u"include", tuple()):
 693         reg_ex = re.compile(str(item).lower())
 694         for job in data:
 695             for build in job:
 696                 for test_id, test in build.iteritems():
 697                     if not re.match(reg_ex, str(test_id).lower()):
 698                         continue
 699                     if y_vals.get(test[u"parent"], None) is None:
 700                         y_vals[test[u"parent"]] = list()
 701                     try:
 702                         if test[u"type"] in (u"SOAK",):
 703                             y_vals[test[u"parent"]]. \
 704                                 append(test[u"throughput"][u"LOWER"])
 705                             test_type = u"SOAK"
 706
 707                         elif test[u"type"] in (u"HOSTSTACK",):
 708                             if u"LDPRELOAD" in test[u"tags"]:
 709                                 y_vals[test[u"parent"]].append(
 710                                     float(
 711                                         test[u"result"][u"bits_per_second"]
 712                                     ) / 1e3
 713                                 )
 714                             elif u"VPPECHO" in test[u"tags"]:
 715                                 y_vals[test[u"parent"]].append(
 716                                     (float(
 717                                         test[u"result"][u"client"][u"tx_data"]
 718                                     ) * 8 / 1e3) /
 719                                     ((float(
 720                                         test[u"result"][u"client"][u"time"]
 721                                     ) +
 722                                       float(
 723                                           test[u"result"][u"server"][u"time"])
 724                                       ) / 2)
 725                                 )
 726                             test_type = u"HOSTSTACK"
 727
 728                         elif test[u"type"] in (u"LDP_NGINX",):
 729                             if u"TCP_CPS" in test[u"tags"]:
 730                                 test_type = u"VSAP_CPS"
 731                                 y_vals[test[u"parent"]].append(
 732                                     test[u"result"][u"cps"]
 733                                 )
 734                             elif u"TCP_RPS" in test[u"tags"]:
 735                                 test_type = u"VSAP_RPS"
 736                                 y_vals[test[u"parent"]].append(
 737                                     test[u"result"][u"rps"]
 738                                 )
 739                             else:
 740                                 continue
 741                         else:
 742                             continue
 743
 744                     except (KeyError, TypeError):
 745                         y_vals[test[u"parent"]].append(None)
 746
 747     # Add None to the lists with missing data
 748     max_len = 0
 749     nr_of_samples = list()
 750     for val in y_vals.values():
 751         if len(val) > max_len:
 752             max_len = len(val)
 753         nr_of_samples.append(len(val))
 754     for val in y_vals.values():
 755         if len(val) < max_len:
 756             val.extend([None for _ in range(max_len - len(val))])
 757
 758     # Add plot traces
 759     traces = list()
 760     df_y = pd.DataFrame(y_vals)
 761     df_y.head()
 762     y_max = list()
 763     for i, col in enumerate(df_y.columns):
 764         tst_name = re.sub(REGEX_NIC, u"",
 765                           col.lower().replace(u'-ndrpdr', u'').
 766                           replace(u'2n1l-', u''))
 767         if test_type in (u"VSAP_CPS", u"VSAP_RPS"):
 768             data_y = [y if y else None for y in df_y[col]]
 769         else:
 770             data_y = [y / 1e6 if y else None for y in df_y[col]]
 771         kwargs = dict(
 772             x=[str(i + 1) + u'.'] * len(df_y[col]),
 773             y=data_y,
 774             name=(
 775                 f"{i + 1}. "
 776                 f"({nr_of_samples[i]:02d} "
 777                 f"run{u's' if nr_of_samples[i] > 1 else u''}) "
 778                 f"{tst_name}"
 779             ),
 780             hoverinfo=u"y+name"
 781         )
 782         if test_type in (u"SOAK", ):
 783             kwargs[u"boxpoints"] = u"all"
 784
 785         traces.append(plgo.Box(**kwargs))
 786
 787         try:
 788             val_max = max(df_y[col])
 789             if val_max:
 790                 y_max.append(int(val_max / 1e6))
 791         except (ValueError, TypeError) as err:
 792             logging.error(repr(err))
 793             continue
 794
 795     try:
 796         # Create plot
 797         layout = deepcopy(plot[u"layout"])
 798         if layout.get(u"title", None):
 799             if test_type in (u"HOSTSTACK", ):
 800                 layout[u"title"] = f"<b>Bandwidth:</b> {layout[u'title']}"
 801             elif test_type == u"VSAP_CPS":
 802                 layout[u"title"] = f"<b>CPS:</b> {layout[u'title']}"
 803                 layout[u"yaxis"][u"title"] = u"<b>Connection Rate [cps]</b>"
 804             elif test_type == u"VSAP_RPS":
 805                 layout[u"title"] = f"<b>RPS:</b> {layout[u'title']}"
 806                 layout[u"yaxis"][u"title"] = u"<b>Connection Rate [rps]</b>"
 807             else:
 808                 layout[u"title"] = f"<b>Tput:</b> {layout[u'title']}"
 809         if y_max and max(y_max) > 1:
 810             layout[u"yaxis"][u"range"] = [0, max(y_max) + 2]
 811         plpl = plgo.Figure(data=traces, layout=layout)
 812
 813         # Export Plot
 814         logging.info(f"    Writing file {plot[u'output-file']}.html.")
 815         ploff.plot(
 816             plpl,
 817             show_link=False,
 818             auto_open=False,
 819             filename=f"{plot[u'output-file']}.html"
 820         )
 821     except PlotlyError as err:
 822         logging.error(
 823             f"   Finished with error: {repr(err)}".replace(u"\n", u" ")
 824         )
 825         return
 826
 827
 828 def plot_ndrpdr_box_name(plot, input_data):
 829     """Generate the plot(s) with algorithm: plot_ndrpdr_box_name
 830     specified in the specification file.
 831
 832     :param plot: Plot to generate.
 833     :param input_data: Data to process.
 834     :type plot: pandas.Series
 835     :type input_data: InputData
 836     """
 837
 838     # Transform the data
 839     logging.info(
 840         f"    Creating data set for the {plot.get(u'type', u'')} "
 841         f"{plot.get(u'title', u'')}."
 842     )
 843     data = input_data.filter_tests_by_name(
 844         plot,
 845         params=[u"throughput", u"gbps", u"parent", u"tags", u"type"]
 846     )
 847     if data is None:
 848         logging.error(u"No data.")
 849         return
 850
 851     if u"-gbps" in plot.get(u"title", u"").lower():
 852         value = u"gbps"
 853         multiplier = 1e6
 854     else:
 855         value = u"throughput"
 856         multiplier = 1.0
 857
 858     test_type = u""
 859
 860     for ttype in plot.get(u"test-type", (u"ndr", u"pdr")):
 861         for core in plot.get(u"core", tuple()):
 862             # Prepare the data for the plot
 863             data_x = list()
 864             data_y = OrderedDict()
 865             data_y_max = list()
 866             idx = 1
 867             for item in plot.get(u"include", tuple()):
 868                 reg_ex = re.compile(str(item.format(core=core)).lower())
 869                 for job in data:
 870                     for build in job:
 871                         for test_id, test in build.iteritems():
 872                             if not re.match(reg_ex, str(test_id).lower()):
 873                                 continue
 874                             if data_y.get(test[u"parent"], None) is None:
 875                                 data_y[test[u"parent"]] = list()
 876                                 test_type = test[u"type"]
 877                                 data_x.append(idx)
 878                                 idx += 1
 879                             try:
 880                                 data_y[test[u"parent"]].append(
 881                                     test[value][ttype.upper()][u"LOWER"] *
 882                                     multiplier
 883                                 )
 884                             except (KeyError, TypeError):
 885                                 pass
 886
 887             # Add plot traces
 888             traces = list()
 889             for idx, (key, vals) in enumerate(data_y.items()):
 890                 name = re.sub(
 891                     REGEX_NIC, u'', key.lower().replace(u'-ndrpdr', u'').
 892                     replace(u'2n1l-', u'')
 893                 )
 894                 traces.append(
 895                     plgo.Box(
 896                         x=[data_x[idx], ] * len(data_x),
 897                         y=[y / 1e6 if y else None for y in vals],
 898                         name=(
 899                             f"{idx+1}."
 900                             f"({len(vals):02d} "
 901                             f"run"
 902                             f"{u's' if len(vals) > 1 else u''}) "
 903                             f"{name}"
 904                         ),
 905                         hoverinfo=u"y+name"
 906                     )
 907                 )
 908                 try:
 909                     data_y_max.append(max(vals))
 910                 except ValueError as err:
 911                     logging.warning(f"No values to use.\n{err!r}")
 912             try:
 913                 # Create plot
 914                 layout = deepcopy(plot[u"layout"])
 915                 if layout.get(u"title", None):
 916                     layout[u"title"] = \
 917                         layout[u'title'].format(core=core, test_type=ttype)
 918                     if test_type in (u"CPS", ):
 919                         layout[u"title"] = f"<b>CPS:</b> {layout[u'title']}"
 920                     else:
 921                         layout[u"title"] = \
 922                             f"<b>Tput:</b> {layout[u'title']}"
 923                 if data_y_max:
 924                     layout[u"yaxis"][u"range"] = [0, max(data_y_max) / 1e6 + 1]
 925                 plpl = plgo.Figure(data=traces, layout=layout)
 926
 927                 # Export Plot
 928                 file_name = (
 929                     f"{plot[u'output-file'].format(core=core, test_type=ttype)}"
 930                     f".html"
 931                 )
 932                 logging.info(f"    Writing file {file_name}")
 933                 ploff.plot(
 934                     plpl,
 935                     show_link=False,
 936                     auto_open=False,
 937                     filename=file_name
 938                 )
 939             except PlotlyError as err:
 940                 logging.error(
 941                     f"   Finished with error: {repr(err)}".replace(u"\n", u" ")
 942                 )
 943
 944
 945 def plot_mrr_box_name(plot, input_data):
 946     """Generate the plot(s) with algorithm: plot_mrr_box_name
 947     specified in the specification file.
 948
 949     :param plot: Plot to generate.
 950     :param input_data: Data to process.
 951     :type plot: pandas.Series
 952     :type input_data: InputData
 953     """
 954
 955     # Transform the data
 956     logging.info(
 957         f"    Creating data set for the {plot.get(u'type', u'')} "
 958         f"{plot.get(u'title', u'')}."
 959     )
 960     data = input_data.filter_tests_by_name(
 961         plot,
 962         params=[u"result", u"parent", u"tags", u"type"]
 963     )
 964     if data is None:
 965         logging.error(u"No data.")
 966         return
 967
 968     for core in plot.get(u"core", tuple()):
 969         # Prepare the data for the plot
 970         data_x = list()
 971         data_names = list()
 972         data_y = list()
 973         data_y_max = list()
 974         idx = 1
 975         for item in plot.get(u"include", tuple()):
 976             reg_ex = re.compile(str(item.format(core=core)).lower())
 977             for job in data:
 978                 for build in job:
 979                     for test_id, test in build.iteritems():
 980                         if not re.match(reg_ex, str(test_id).lower()):
 981                             continue
 982                         try:
 983                             data_x.append(idx)
 984                             name = re.sub(
 985                                 REGEX_NIC, u'', test[u'parent'].lower().
 986                                 replace(u'-mrr', u'').replace(u'2n1l-', u'')
 987                             )
 988                             data_y.append(test[u"result"][u"samples"])
 989                             data_names.append(
 990                                 f"{idx}."
 991                                 f"({len(data_y[-1]):02d} "
 992                                 f"run{u's' if len(data_y[-1]) > 1 else u''}) "
 993                                 f"{name}"
 994                             )
 995                             data_y_max.append(max(data_y[-1]))
 996                             idx += 1
 997                         except (KeyError, TypeError):
 998                             pass
 999
1000         # Add plot traces
1001         traces = list()
1002         for idx, x_item in enumerate(data_x):
1003             traces.append(
1004                 plgo.Box(
1005                     x=[x_item, ] * len(data_y[idx]),
1006                     y=data_y[idx],
1007                     name=data_names[idx],
1008                     hoverinfo=u"y+name"
1009                 )
1010             )
1011
1012         try:
1013             # Create plot
1014             layout = deepcopy(plot[u"layout"])
1015             if layout.get(u"title", None):
1016                 layout[u"title"] = (
1017                     f"<b>Tput:</b> {layout[u'title'].format(core=core)}"
1018                 )
1019             if data_y_max:
1020                 layout[u"yaxis"][u"range"] = [0, max(data_y_max) + 1]
1021             plpl = plgo.Figure(data=traces, layout=layout)
1022
1023             # Export Plot
1024             file_name = f"{plot[u'output-file'].format(core=core)}.html"
1025             logging.info(f"    Writing file {file_name}")
1026             ploff.plot(
1027                 plpl,
1028                 show_link=False,
1029                 auto_open=False,
1030                 filename=file_name
1031             )
1032         except PlotlyError as err:
1033             logging.error(
1034                 f"   Finished with error: {repr(err)}".replace(u"\n", u" ")
1035             )
1036
1037
1038 def plot_tsa_name(plot, input_data):
1039     """Generate the plot(s) with algorithm:
1040     plot_tsa_name
1041     specified in the specification file.
1042
1043     :param plot: Plot to generate.
1044     :param input_data: Data to process.
1045     :type plot: pandas.Series
1046     :type input_data: InputData
1047     """
1048
1049     # Transform the data
1050     plot_title = plot.get(u"title", u"")
1051     logging.info(
1052         f"    Creating data set for the {plot.get(u'type', u'')} {plot_title}."
1053     )
1054     data = input_data.filter_tests_by_name(
1055         plot,
1056         params=[u"throughput", u"gbps", u"parent", u"tags", u"type"]
1057     )
1058     if data is None:
1059         logging.error(u"No data.")
1060         return
1061
1062     plot_title = plot_title.lower()
1063
1064     if u"-gbps" in plot_title:
1065         value = u"gbps"
1066         h_unit = u"Gbps"
1067         multiplier = 1e6
1068     else:
1069         value = u"throughput"
1070         h_unit = u"Mpps"
1071         multiplier = 1.0
1072
1073     for ttype in plot.get(u"test-type", (u"ndr", u"pdr")):
1074         y_vals = OrderedDict()
1075         for item in plot.get(u"include", tuple()):
1076             reg_ex = re.compile(str(item).lower())
1077             for job in data:
1078                 for build in job:
1079                     for test_id, test in build.iteritems():
1080                         if re.match(reg_ex, str(test_id).lower()):
1081                             if y_vals.get(test[u"parent"], None) is None:
1082                                 y_vals[test[u"parent"]] = {
1083                                     u"1": list(),
1084                                     u"2": list(),
1085                                     u"4": list()
1086                                 }
1087                             try:
1088                                 if test[u"type"] not in (u"NDRPDR", u"CPS"):
1089                                     continue
1090
1091                                 if u"1C" in test[u"tags"]:
1092                                     y_vals[test[u"parent"]][u"1"].append(
1093                                         test[value][ttype.upper()][u"LOWER"] *
1094                                         multiplier
1095                                     )
1096                                 elif u"2C" in test[u"tags"]:
1097                                     y_vals[test[u"parent"]][u"2"].append(
1098                                         test[value][ttype.upper()][u"LOWER"] *
1099                                         multiplier
1100                                     )
1101                                 elif u"4C" in test[u"tags"]:
1102                                     y_vals[test[u"parent"]][u"4"].append(
1103                                         test[value][ttype.upper()][u"LOWER"] *
1104                                         multiplier
1105                                     )
1106                             except (KeyError, TypeError):
1107                                 pass
1108
1109         if not y_vals:
1110             logging.warning(f"No data for the plot {plot.get(u'title', u'')}")
1111             return
1112
1113         y_1c_max = dict()
1114         for test_name, test_vals in y_vals.items():
1115             for key, test_val in test_vals.items():
1116                 if test_val:
1117                     avg_val = sum(test_val) / len(test_val)
1118                     y_vals[test_name][key] = [avg_val, len(test_val)]
1119                     ideal = avg_val / (int(key) * 1e6)
1120                     if test_name not in y_1c_max or ideal > y_1c_max[test_name]:
1121                         y_1c_max[test_name] = ideal
1122
1123         vals = OrderedDict()
1124         y_max = list()
1125         nic_limit = 0
1126         lnk_limit = 0
1127         pci_limit = 0
1128         for test_name, test_vals in y_vals.items():
1129             try:
1130                 if test_vals[u"1"][1]:
1131                     name = re.sub(
1132                         REGEX_NIC,
1133                         u"",
1134                         test_name.replace(u'-ndrpdr', u'').
1135                         replace(u'2n1l-', u'')
1136                     )
1137                     vals[name] = OrderedDict()
1138                     y_val_1 = test_vals[u"1"][0] / 1e6
1139                     y_val_2 = test_vals[u"2"][0] / 1e6 if test_vals[u"2"][0] \
1140                         else None
1141                     y_val_4 = test_vals[u"4"][0] / 1e6 if test_vals[u"4"][0] \
1142                         else None
1143
1144                     vals[name][u"val"] = [y_val_1, y_val_2, y_val_4]
1145                     vals[name][u"rel"] = [1.0, None, None]
1146                     vals[name][u"ideal"] = [
1147                         y_1c_max[test_name],
1148                         y_1c_max[test_name] * 2,
1149                         y_1c_max[test_name] * 4
1150                     ]
1151                     vals[name][u"diff"] = [
1152                         (y_val_1 - y_1c_max[test_name]) * 100 / y_val_1,
1153                         None,
1154                         None
1155                     ]
1156                     vals[name][u"count"] = [
1157                         test_vals[u"1"][1],
1158                         test_vals[u"2"][1],
1159                         test_vals[u"4"][1]
1160                     ]
1161
1162                     try:
1163                         val_max = max(vals[name][u"val"])
1164                     except ValueError as err:
1165                         logging.error(repr(err))
1166                         continue
1167                     if val_max:
1168                         y_max.append(val_max)
1169
1170                     if y_val_2:
1171                         vals[name][u"rel"][1] = round(y_val_2 / y_val_1, 2)
1172                         vals[name][u"diff"][1] = \
1173                             (y_val_2 - vals[name][u"ideal"][1]) * 100 / y_val_2
1174                     if y_val_4:
1175                         vals[name][u"rel"][2] = round(y_val_4 / y_val_1, 2)
1176                         vals[name][u"diff"][2] = \
1177                             (y_val_4 - vals[name][u"ideal"][2]) * 100 / y_val_4
1178             except IndexError as err:
1179                 logging.warning(f"No data for {test_name}")
1180                 logging.warning(repr(err))
1181
1182             # Limits:
1183             if u"x520" in test_name:
1184                 limit = plot[u"limits"][u"nic"][u"x520"]
1185             elif u"x710" in test_name:
1186                 limit = plot[u"limits"][u"nic"][u"x710"]
1187             elif u"xxv710" in test_name:
1188                 limit = plot[u"limits"][u"nic"][u"xxv710"]
1189             elif u"xl710" in test_name:
1190                 limit = plot[u"limits"][u"nic"][u"xl710"]
1191             elif u"x553" in test_name:
1192                 limit = plot[u"limits"][u"nic"][u"x553"]
1193             elif u"cx556a" in test_name:
1194                 limit = plot[u"limits"][u"nic"][u"cx556a"]
1195             elif u"e810cq" in test_name:
1196                 limit = plot[u"limits"][u"nic"][u"e810cq"]
1197             else:
1198                 limit = 0
1199             if limit > nic_limit:
1200                 nic_limit = limit
1201
1202             mul = 2 if u"ge2p" in test_name else 1
1203             if u"10ge" in test_name:
1204                 limit = plot[u"limits"][u"link"][u"10ge"] * mul
1205             elif u"25ge" in test_name:
1206                 limit = plot[u"limits"][u"link"][u"25ge"] * mul
1207             elif u"40ge" in test_name:
1208                 limit = plot[u"limits"][u"link"][u"40ge"] * mul
1209             elif u"100ge" in test_name:
1210                 limit = plot[u"limits"][u"link"][u"100ge"] * mul
1211             else:
1212                 limit = 0
1213             if limit > lnk_limit:
1214                 lnk_limit = limit
1215
1216             if u"cx556a" in test_name:
1217                 limit = plot[u"limits"][u"pci"][u"pci-g3-x8"]
1218             else:
1219                 limit = plot[u"limits"][u"pci"][u"pci-g3-x16"]
1220             if limit > pci_limit:
1221                 pci_limit = limit
1222
1223         traces = list()
1224         annotations = list()
1225         x_vals = [1, 2, 4]
1226
1227         # Limits:
1228         if u"-gbps" not in plot_title and u"-cps-" not in plot_title:
1229             nic_limit /= 1e6
1230             lnk_limit /= 1e6
1231             pci_limit /= 1e6
1232             min_limit = min((nic_limit, lnk_limit, pci_limit))
1233             if nic_limit == min_limit:
1234                 traces.append(plgo.Scatter(
1235                     x=x_vals,
1236                     y=[nic_limit, ] * len(x_vals),
1237                     name=f"NIC: {nic_limit:.2f}Mpps",
1238                     showlegend=False,
1239                     mode=u"lines",
1240                     line=dict(
1241                         dash=u"dot",
1242                         color=COLORS[-1],
1243                         width=1),
1244                     hoverinfo=u"none"
1245                 ))
1246                 annotations.append(dict(
1247                     x=1,
1248                     y=nic_limit,
1249                     xref=u"x",
1250                     yref=u"y",
1251                     xanchor=u"left",
1252                     yanchor=u"bottom",
1253                     text=f"NIC: {nic_limit:.2f}Mpps",
1254                     font=dict(
1255                         size=14,
1256                         color=COLORS[-1],
1257                     ),
1258                     align=u"left",
1259                     showarrow=False
1260                 ))
1261                 y_max.append(nic_limit)
1262             elif lnk_limit == min_limit:
1263                 traces.append(plgo.Scatter(
1264                     x=x_vals,
1265                     y=[lnk_limit, ] * len(x_vals),
1266                     name=f"Link: {lnk_limit:.2f}Mpps",
1267                     showlegend=False,
1268                     mode=u"lines",
1269                     line=dict(
1270                         dash=u"dot",
1271                         color=COLORS[-1],
1272                         width=1),
1273                     hoverinfo=u"none"
1274                 ))
1275                 annotations.append(dict(
1276                     x=1,
1277                     y=lnk_limit,
1278                     xref=u"x",
1279                     yref=u"y",
1280                     xanchor=u"left",
1281                     yanchor=u"bottom",
1282                     text=f"Link: {lnk_limit:.2f}Mpps",
1283                     font=dict(
1284                         size=14,
1285                         color=COLORS[-1],
1286                     ),
1287                     align=u"left",
1288                     showarrow=False
1289                 ))
1290                 y_max.append(lnk_limit)
1291             elif pci_limit == min_limit:
1292                 traces.append(plgo.Scatter(
1293                     x=x_vals,
1294                     y=[pci_limit, ] * len(x_vals),
1295                     name=f"PCIe: {pci_limit:.2f}Mpps",
1296                     showlegend=False,
1297                     mode=u"lines",
1298                     line=dict(
1299                         dash=u"dot",
1300                         color=COLORS[-1],
1301                         width=1),
1302                     hoverinfo=u"none"
1303                 ))
1304                 annotations.append(dict(
1305                     x=1,
1306                     y=pci_limit,
1307                     xref=u"x",
1308                     yref=u"y",
1309                     xanchor=u"left",
1310                     yanchor=u"bottom",
1311                     text=f"PCIe: {pci_limit:.2f}Mpps",
1312                     font=dict(
1313                         size=14,
1314                         color=COLORS[-1],
1315                     ),
1316                     align=u"left",
1317                     showarrow=False
1318                 ))
1319                 y_max.append(pci_limit)
1320
1321         # Perfect and measured:
1322         cidx = 0
1323         for name, val in vals.items():
1324             hovertext = list()
1325             try:
1326                 for idx in range(len(val[u"val"])):
1327                     htext = ""
1328                     if isinstance(val[u"val"][idx], float):
1329                         htext += (
1330                             f"No. of Runs: {val[u'count'][idx]}<br>"
1331                             f"Mean: {val[u'val'][idx]:.2f}{h_unit}<br>"
1332                         )
1333                     if isinstance(val[u"diff"][idx], float):
1334                         htext += f"Diff: {round(val[u'diff'][idx]):.0f}%<br>"
1335                     if isinstance(val[u"rel"][idx], float):
1336                         htext += f"Speedup: {val[u'rel'][idx]:.2f}"
1337                     hovertext.append(htext)
1338                 traces.append(
1339                     plgo.Scatter(
1340                         x=x_vals,
1341                         y=val[u"val"],
1342                         name=name,
1343                         legendgroup=name,
1344                         mode=u"lines+markers",
1345                         line=dict(
1346                             color=COLORS[cidx],
1347                             width=2),
1348                         marker=dict(
1349                             symbol=u"circle",
1350                             size=10
1351                         ),
1352                         text=hovertext,
1353                         hoverinfo=u"text+name"
1354                     )
1355                 )
1356                 traces.append(
1357                     plgo.Scatter(
1358                         x=x_vals,
1359                         y=val[u"ideal"],
1360                         name=f"{name} perfect",
1361                         legendgroup=name,
1362                         showlegend=False,
1363                         mode=u"lines",
1364                         line=dict(
1365                             color=COLORS[cidx],
1366                             width=2,
1367                             dash=u"dash"),
1368                         text=[f"Perfect: {y:.2f}Mpps" for y in val[u"ideal"]],
1369                         hoverinfo=u"text"
1370                     )
1371                 )
1372                 cidx += 1
1373             except (IndexError, ValueError, KeyError) as err:
1374                 logging.warning(f"No data for {name}\n{repr(err)}")
1375
1376         try:
1377             # Create plot
1378             file_name = f"{plot[u'output-file'].format(test_type=ttype)}.html"
1379             logging.info(f"    Writing file {file_name}")
1380             layout = deepcopy(plot[u"layout"])
1381             if layout.get(u"title", None):
1382                 layout[u"title"] = (
1383                     f"<b>Speedup Multi-core:</b> "
1384                     f"{layout[u'title'].format(test_type=ttype)}"
1385                 )
1386             layout[u"yaxis"][u"range"] = [0, int(max(y_max) * 1.1)]
1387             layout[u"annotations"].extend(annotations)
1388             plpl = plgo.Figure(data=traces, layout=layout)
1389
1390             # Export Plot
1391             ploff.plot(
1392                 plpl,
1393                 show_link=False,
1394                 auto_open=False,
1395                 filename=file_name
1396             )
1397         except PlotlyError as err:
1398             logging.error(
1399                 f"   Finished with error: {repr(err)}".replace(u"\n", u" ")
1400             )
1401
1402
1403 def plot_http_server_perf_box(plot, input_data):
1404     """Generate the plot(s) with algorithm: plot_http_server_perf_box
1405     specified in the specification file.
1406
1407     :param plot: Plot to generate.
1408     :param input_data: Data to process.
1409     :type plot: pandas.Series
1410     :type input_data: InputData
1411     """
1412
1413     # Transform the data
1414     logging.info(
1415         f"    Creating the data set for the {plot.get(u'type', u'')} "
1416         f"{plot.get(u'title', u'')}."
1417     )
1418     data = input_data.filter_data(plot)
1419     if data is None:
1420         logging.error(u"No data.")
1421         return
1422
1423     # Prepare the data for the plot
1424     y_vals = dict()
1425     for job in data:
1426         for build in job:
1427             for test in build:
1428                 if y_vals.get(test[u"name"], None) is None:
1429                     y_vals[test[u"name"]] = list()
1430                 try:
1431                     y_vals[test[u"name"]].append(test[u"result"])
1432                 except (KeyError, TypeError):
1433                     y_vals[test[u"name"]].append(None)
1434
1435     # Add None to the lists with missing data
1436     max_len = 0
1437     nr_of_samples = list()
1438     for val in y_vals.values():
1439         if len(val) > max_len:
1440             max_len = len(val)
1441         nr_of_samples.append(len(val))
1442     for val in y_vals.values():
1443         if len(val) < max_len:
1444             val.extend([None for _ in range(max_len - len(val))])
1445
1446     # Add plot traces
1447     traces = list()
1448     df_y = pd.DataFrame(y_vals)
1449     df_y.head()
1450     for i, col in enumerate(df_y.columns):
1451         name = \
1452             f"{i + 1}. " \
1453             f"({nr_of_samples[i]:02d} " \
1454             f"run{u's' if nr_of_samples[i] > 1 else u''}) " \
1455             f"{col.lower().replace(u'-ndrpdr', u'')}"
1456         if len(name) > 50:
1457             name_lst = name.split(u'-')
1458             name = u""
1459             split_name = True
1460             for segment in name_lst:
1461                 if (len(name) + len(segment) + 1) > 50 and split_name:
1462                     name += u"<br>    "
1463                     split_name = False
1464                 name += segment + u'-'
1465             name = name[:-1]
1466
1467         traces.append(plgo.Box(x=[str(i + 1) + u'.'] * len(df_y[col]),
1468                                y=df_y[col],
1469                                name=name,
1470                                **plot[u"traces"]))
1471     try:
1472         # Create plot
1473         plpl = plgo.Figure(data=traces, layout=plot[u"layout"])
1474
1475         # Export Plot
1476         logging.info(
1477             f"    Writing file {plot[u'output-file']}"
1478             f"{plot[u'output-file-type']}."
1479         )
1480         ploff.plot(
1481             plpl,
1482             show_link=False,
1483             auto_open=False,
1484             filename=f"{plot[u'output-file']}{plot[u'output-file-type']}"
1485         )
1486     except PlotlyError as err:
1487         logging.error(
1488             f"   Finished with error: {repr(err)}".replace(u"\n", u" ")
1489         )
1490         return
1491
1492
1493 def plot_nf_heatmap(plot, input_data):
1494     """Generate the plot(s) with algorithm: plot_nf_heatmap
1495     specified in the specification file.
1496
1497     :param plot: Plot to generate.
1498     :param input_data: Data to process.
1499     :type plot: pandas.Series
1500     :type input_data: InputData
1501     """
1502
1503     def sort_by_int(value):
1504         """Makes possible to sort a list of strings which represent integers.
1505
1506         :param value: Integer as a string.
1507         :type value: str
1508         :returns: Integer representation of input parameter 'value'.
1509         :rtype: int
1510         """
1511         return int(value)
1512
1513     regex_cn = re.compile(r'^(\d*)R(\d*)C$')
1514     regex_test_name = re.compile(r'^.*-(\d+ch|\d+pl)-'
1515                                  r'(\d+mif|\d+vh)-'
1516                                  r'(\d+vm\d+t|\d+dcr\d+t|\d+dcr\d+c).*$')
1517     vals = dict()
1518
1519     # Transform the data
1520     logging.info(
1521         f"    Creating the data set for the {plot.get(u'type', u'')} "
1522         f"{plot.get(u'title', u'')}."
1523     )
1524     in_data = input_data.filter_tests_by_name(
1525         plot,
1526         continue_on_error=True,
1527         params=[u"throughput", u"result", u"name", u"tags", u"type"]
1528     )
1529     if in_data is None or in_data.empty:
1530         logging.error(u"No data.")
1531         return
1532
1533     for ttype in plot.get(u"test-type", (u"ndr", u"pdr")):
1534         for core in plot.get(u"core", tuple()):
1535             for item in plot.get(u"include", tuple()):
1536                 reg_ex = re.compile(str(item.format(core=core)).lower())
1537                 for job in in_data:
1538                     for build in job:
1539                         for test_id, test in build.iteritems():
1540                             if not re.match(reg_ex, str(test_id).lower()):
1541                                 continue
1542                             for tag in test[u"tags"]:
1543                                 groups = re.search(regex_cn, tag)
1544                                 if groups:
1545                                     chain = str(groups.group(1))
1546                                     node = str(groups.group(2))
1547                                     break
1548                             else:
1549                                 continue
1550                             groups = re.search(regex_test_name, test[u"name"])
1551                             if groups and len(groups.groups()) == 3:
1552                                 hover_name = (
1553                                     f"{str(groups.group(1))}-"
1554                                     f"{str(groups.group(2))}-"
1555                                     f"{str(groups.group(3))}"
1556                                 )
1557                             else:
1558                                 hover_name = u""
1559                             if vals.get(chain, None) is None:
1560                                 vals[chain] = dict()
1561                             if vals[chain].get(node, None) is None:
1562                                 vals[chain][node] = dict(
1563                                     name=hover_name,
1564                                     vals=list(),
1565                                     nr=None,
1566                                     mean=None,
1567                                     stdev=None
1568                                 )
1569                             try:
1570                                 if ttype == u"mrr":
1571                                     result = test[u"result"][u"receive-rate"]
1572                                 elif ttype == u"pdr":
1573                                     result = \
1574                                         test[u"throughput"][u"PDR"][u"LOWER"]
1575                                 elif ttype == u"ndr":
1576                                     result = \
1577                                         test[u"throughput"][u"NDR"][u"LOWER"]
1578                                 else:
1579                                     result = None
1580                             except TypeError:
1581                                 result = None
1582
1583                             if result:
1584                                 vals[chain][node][u"vals"].append(result)
1585
1586             if not vals:
1587                 logging.error(u"No data.")
1588                 return
1589
1590             txt_chains = list()
1591             txt_nodes = list()
1592             for key_c in vals:
1593                 txt_chains.append(key_c)
1594                 for key_n in vals[key_c].keys():
1595                     txt_nodes.append(key_n)
1596                     if vals[key_c][key_n][u"vals"]:
1597                         vals[key_c][key_n][u"nr"] = \
1598                             len(vals[key_c][key_n][u"vals"])
1599                         vals[key_c][key_n][u"mean"] = \
1600                             round(mean(vals[key_c][key_n][u"vals"]) / 1e6, 1)
1601                         vals[key_c][key_n][u"stdev"] = \
1602                             round(stdev(vals[key_c][key_n][u"vals"]) / 1e6, 1)
1603             txt_nodes = list(set(txt_nodes))
1604
1605             txt_chains = sorted(txt_chains, key=sort_by_int)
1606             txt_nodes = sorted(txt_nodes, key=sort_by_int)
1607
1608             chains = [i + 1 for i in range(len(txt_chains))]
1609             nodes = [i + 1 for i in range(len(txt_nodes))]
1610
1611             data = [list() for _ in range(len(chains))]
1612             for chain in chains:
1613                 for node in nodes:
1614                     try:
1615                         val = vals[txt_chains[chain - 1]] \
1616                             [txt_nodes[node - 1]][u"mean"]
1617                     except (KeyError, IndexError):
1618                         val = None
1619                     data[chain - 1].append(val)
1620
1621             # Color scales:
1622             my_green = [[0.0, u"rgb(235, 249, 242)"],
1623                         [1.0, u"rgb(45, 134, 89)"]]
1624
1625             my_blue = [[0.0, u"rgb(236, 242, 248)"],
1626                        [1.0, u"rgb(57, 115, 172)"]]
1627
1628             my_grey = [[0.0, u"rgb(230, 230, 230)"],
1629                        [1.0, u"rgb(102, 102, 102)"]]
1630
1631             hovertext = list()
1632             annotations = list()
1633
1634             text = (u"Test: {name}<br>"
1635                     u"Runs: {nr}<br>"
1636                     u"Thput: {val}<br>"
1637                     u"StDev: {stdev}")
1638
1639             for chain, _ in enumerate(txt_chains):
1640                 hover_line = list()
1641                 for node, _ in enumerate(txt_nodes):
1642                     if data[chain][node] is not None:
1643                         annotations.append(
1644                             dict(
1645                                 x=node+1,
1646                                 y=chain+1,
1647                                 xref=u"x",
1648                                 yref=u"y",
1649                                 xanchor=u"center",
1650                                 yanchor=u"middle",
1651                                 text=str(data[chain][node]),
1652                                 font=dict(
1653                                     size=14,
1654                                 ),
1655                                 align=u"center",
1656                                 showarrow=False
1657                             )
1658                         )
1659                         hover_line.append(text.format(
1660                             name=vals[txt_chains[chain]][txt_nodes[node]]
1661                             [u"name"],
1662                             nr=vals[txt_chains[chain]][txt_nodes[node]][u"nr"],
1663                             val=data[chain][node],
1664                             stdev=vals[txt_chains[chain]][txt_nodes[node]]
1665                             [u"stdev"]
1666                         ))
1667                 hovertext.append(hover_line)
1668
1669             traces = [
1670                 plgo.Heatmap(
1671                     x=nodes,
1672                     y=chains,
1673                     z=data,
1674                     colorbar=dict(
1675                         title=plot.get(u"z-axis", u"{test_type}").
1676                         format(test_type=ttype.upper()),
1677                         titleside=u"right",
1678                         titlefont=dict(
1679                             size=16
1680                         ),
1681                         tickfont=dict(
1682                             size=16,
1683                         ),
1684                         tickformat=u".1f",
1685                         yanchor=u"bottom",
1686                         y=-0.02,
1687                         len=0.925,
1688                     ),
1689                     showscale=True,
1690                     colorscale=my_green,
1691                     text=hovertext,
1692                     hoverinfo=u"text"
1693                 )
1694             ]
1695
1696             for idx, item in enumerate(txt_nodes):
1697                 # X-axis, numbers:
1698                 annotations.append(
1699                     dict(
1700                         x=idx+1,
1701                         y=0.05,
1702                         xref=u"x",
1703                         yref=u"y",
1704                         xanchor=u"center",
1705                         yanchor=u"top",
1706                         text=item,
1707                         font=dict(
1708                             size=16,
1709                         ),
1710                         align=u"center",
1711                         showarrow=False
1712                     )
1713                 )
1714             for idx, item in enumerate(txt_chains):
1715                 # Y-axis, numbers:
1716                 annotations.append(
1717                     dict(
1718                         x=0.35,
1719                         y=idx+1,
1720                         xref=u"x",
1721                         yref=u"y",
1722                         xanchor=u"right",
1723                         yanchor=u"middle",
1724                         text=item,
1725                         font=dict(
1726                             size=16,
1727                         ),
1728                         align=u"center",
1729                         showarrow=False
1730                     )
1731                 )
1732             # X-axis, title:
1733             annotations.append(
1734                 dict(
1735                     x=0.55,
1736                     y=-0.15,
1737                     xref=u"paper",
1738                     yref=u"y",
1739                     xanchor=u"center",
1740                     yanchor=u"bottom",
1741                     text=plot.get(u"x-axis", u""),
1742                     font=dict(
1743                         size=16,
1744                     ),
1745                     align=u"center",
1746                     showarrow=False
1747                 )
1748             )
1749             # Y-axis, title:
1750             annotations.append(
1751                 dict(
1752                     x=-0.1,
1753                     y=0.5,
1754                     xref=u"x",
1755                     yref=u"paper",
1756                     xanchor=u"center",
1757                     yanchor=u"middle",
1758                     text=plot.get(u"y-axis", u""),
1759                     font=dict(
1760                         size=16,
1761                     ),
1762                     align=u"center",
1763                     textangle=270,
1764                     showarrow=False
1765                 )
1766             )
1767             updatemenus = list([
1768                 dict(
1769                     x=1.0,
1770                     y=0.0,
1771                     xanchor=u"right",
1772                     yanchor=u"bottom",
1773                     direction=u"up",
1774                     buttons=list([
1775                         dict(
1776                             args=[
1777                                 {
1778                                     u"colorscale": [my_green, ],
1779                                     u"reversescale": False
1780                                 }
1781                             ],
1782                             label=u"Green",
1783                             method=u"update"
1784                         ),
1785                         dict(
1786                             args=[
1787                                 {
1788                                     u"colorscale": [my_blue, ],
1789                                     u"reversescale": False
1790                                 }
1791                             ],
1792                             label=u"Blue",
1793                             method=u"update"
1794                         ),
1795                         dict(
1796                             args=[
1797                                 {
1798                                     u"colorscale": [my_grey, ],
1799                                     u"reversescale": False
1800                                 }
1801                             ],
1802                             label=u"Grey",
1803                             method=u"update"
1804                         )
1805                     ])
1806                 )
1807             ])
1808
1809             try:
1810                 layout = deepcopy(plot[u"layout"])
1811             except KeyError as err:
1812                 logging.error(
1813                     f"Finished with error: No layout defined\n{repr(err)}"
1814                 )
1815                 return
1816
1817             layout[u"annotations"] = annotations
1818             layout[u'updatemenus'] = updatemenus
1819             if layout.get(u"title", None):
1820                 layout[u"title"] = layout[u'title'].replace(u"test_type", ttype)
1821
1822             try:
1823                 # Create plot
1824                 plpl = plgo.Figure(data=traces, layout=layout)
1825
1826                 # Export Plot
1827                 file_name = (
1828                     f"{plot[u'output-file'].format(core=core, test_type=ttype)}"
1829                     f".html"
1830                 )
1831                 logging.info(f"    Writing file {file_name}")
1832                 ploff.plot(
1833                     plpl,
1834                     show_link=False,
1835                     auto_open=False,
1836                     filename=file_name
1837                 )
1838             except PlotlyError as err:
1839                 logging.error(
1840                     f"   Finished with error: {repr(err)}".replace(u"\n", u" ")
1841                 )