resources/tools/presentation/generator_plots.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate plots.
  15 """
  16
  17
  18 import re
  19 import logging
  20
  21 from collections import OrderedDict
  22 from copy import deepcopy
  23
  24 import hdrh.histogram
  25 import hdrh.codec
  26 import pandas as pd
  27 import plotly.offline as ploff
  28 import plotly.graph_objs as plgo
  29
  30 from plotly.subplots import make_subplots
  31 from plotly.exceptions import PlotlyError
  32
  33 from pal_utils import mean, stdev
  34
  35
  36 COLORS = [u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink",
  37           u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black",
  38           u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson",
  39           u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod",
  40           u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon",
  41           u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey"]
  42
  43 REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*-')
  44
  45
  46 def generate_plots(spec, data):
  47     """Generate all plots specified in the specification file.
  48
  49     :param spec: Specification read from the specification file.
  50     :param data: Data to process.
  51     :type spec: Specification
  52     :type data: InputData
  53     """
  54
  55     generator = {
  56         u"plot_nf_reconf_box_name": plot_nf_reconf_box_name,
  57         u"plot_perf_box_name": plot_perf_box_name,
  58         u"plot_lat_err_bars_name": plot_lat_err_bars_name,
  59         u"plot_tsa_name": plot_tsa_name,
  60         u"plot_http_server_perf_box": plot_http_server_perf_box,
  61         u"plot_nf_heatmap": plot_nf_heatmap,
  62         u"plot_lat_hdrh_bar_name": plot_lat_hdrh_bar_name,
  63         u"plot_lat_hdrh_percentile": plot_lat_hdrh_percentile
  64     }
  65
  66     logging.info(u"Generating the plots ...")
  67     for index, plot in enumerate(spec.plots):
  68         try:
  69             logging.info(f"  Plot nr {index + 1}: {plot.get(u'title', u'')}")
  70             plot[u"limits"] = spec.configuration[u"limits"]
  71             generator[plot[u"algorithm"]](plot, data)
  72             logging.info(u"  Done.")
  73         except NameError as err:
  74             logging.error(
  75                 f"Probably algorithm {plot[u'algorithm']} is not defined: "
  76                 f"{repr(err)}"
  77             )
  78     logging.info(u"Done.")
  79
  80
  81 def plot_lat_hdrh_percentile(plot, input_data):
  82     """Generate the plot(s) with algorithm: plot_lat_hdrh_percentile
  83     specified in the specification file.
  84
  85     :param plot: Plot to generate.
  86     :param input_data: Data to process.
  87     :type plot: pandas.Series
  88     :type input_data: InputData
  89     """
  90
  91     # Transform the data
  92     plot_title = plot.get(u"title", u"")
  93     logging.info(
  94         f"    Creating the data set for the {plot.get(u'type', u'')} "
  95         f"{plot_title}."
  96     )
  97     data = input_data.filter_tests_by_name(
  98         plot, params=[u"latency", u"parent", u"tags", u"type"])
  99     if data is None or len(data[0][0]) == 0:
 100         logging.error(u"No data.")
 101         return
 102
 103     fig = plgo.Figure()
 104
 105     # Prepare the data for the plot
 106     directions = [u"W-E", u"E-W"]
 107     for color, test in enumerate(data[0][0]):
 108         try:
 109             if test[u"type"] in (u"NDRPDR",):
 110                 if u"-pdr" in plot_title.lower():
 111                     ttype = u"PDR"
 112                 elif u"-ndr" in plot_title.lower():
 113                     ttype = u"NDR"
 114                 else:
 115                     logging.warning(f"Invalid test type: {test[u'type']}")
 116                     continue
 117                 name = re.sub(REGEX_NIC, u"", test[u"parent"].
 118                               replace(u'-ndrpdr', u'').
 119                               replace(u'2n1l-', u'').
 120                               replace(u'avf-', u''))
 121                 for idx, direction in enumerate(
 122                         (u"direction1", u"direction2", )):
 123                     try:
 124                         hdr_lat = test[u"latency"][ttype][direction][u"hdrh"]
 125                         # TODO: Workaround, HDRH data must be aligned to 4
 126                         #       bytes, remove when not needed.
 127                         hdr_lat += u"=" * (len(hdr_lat) % 4)
 128                         xaxis = list()
 129                         yaxis = list()
 130                         hovertext = list()
 131                         decoded = hdrh.histogram.HdrHistogram.decode(hdr_lat)
 132                         for item in decoded.get_recorded_iterator():
 133                             percentile = item.percentile_level_iterated_to
 134                             if percentile != 100.0:
 135                                 xaxis.append(100.0 / (100.0 - percentile))
 136                                 yaxis.append(item.value_iterated_to)
 137                                 hovertext.append(
 138                                     f"Test: {name}<br>"
 139                                     f"Direction: {directions[idx]}<br>"
 140                                     f"Percentile: {percentile:.5f}%<br>"
 141                                     f"Latency: {item.value_iterated_to}uSec"
 142                                 )
 143                         fig.add_trace(
 144                             plgo.Scatter(
 145                                 x=xaxis,
 146                                 y=yaxis,
 147                                 name=name,
 148                                 mode=u"lines",
 149                                 legendgroup=name,
 150                                 showlegend=bool(idx),
 151                                 line=dict(
 152                                     color=COLORS[color]
 153                                 ),
 154                                 hovertext=hovertext,
 155                                 hoverinfo=u"text"
 156                             )
 157                         )
 158                     except hdrh.codec.HdrLengthException as err:
 159                         logging.warning(
 160                             f"No or invalid data for HDRHistogram for the test "
 161                             f"{name}\n{err}"
 162                         )
 163                         continue
 164             else:
 165                 logging.warning(f"Invalid test type: {test[u'type']}")
 166                 continue
 167         except (ValueError, KeyError) as err:
 168             logging.warning(repr(err))
 169
 170     layout = deepcopy(plot[u"layout"])
 171
 172     layout[u"title"][u"text"] = \
 173         f"<b>Latency:</b> {plot.get(u'graph-title', u'')}"
 174     fig[u"layout"].update(layout)
 175
 176     # Create plot
 177     file_type = plot.get(u"output-file-type", u".html")
 178     logging.info(f"    Writing file {plot[u'output-file']}{file_type}.")
 179     try:
 180         # Export Plot
 181         ploff.plot(fig, show_link=False, auto_open=False,
 182                    filename=f"{plot[u'output-file']}{file_type}")
 183     except PlotlyError as err:
 184         logging.error(f"   Finished with error: {repr(err)}")
 185
 186
 187 def plot_lat_hdrh_bar_name(plot, input_data):
 188     """Generate the plot(s) with algorithm: plot_lat_hdrh_bar_name
 189     specified in the specification file.
 190
 191     :param plot: Plot to generate.
 192     :param input_data: Data to process.
 193     :type plot: pandas.Series
 194     :type input_data: InputData
 195     """
 196
 197     # Transform the data
 198     plot_title = plot.get(u"title", u"")
 199     logging.info(
 200         f"    Creating the data set for the {plot.get(u'type', u'')} "
 201         f"{plot_title}."
 202     )
 203     data = input_data.filter_tests_by_name(
 204         plot, params=[u"latency", u"parent", u"tags", u"type"])
 205     if data is None or len(data[0][0]) == 0:
 206         logging.error(u"No data.")
 207         return
 208
 209     # Prepare the data for the plot
 210     directions = [u"W-E", u"E-W"]
 211     tests = list()
 212     traces = list()
 213     for idx_row, test in enumerate(data[0][0]):
 214         try:
 215             if test[u"type"] in (u"NDRPDR",):
 216                 if u"-pdr" in plot_title.lower():
 217                     ttype = u"PDR"
 218                 elif u"-ndr" in plot_title.lower():
 219                     ttype = u"NDR"
 220                 else:
 221                     logging.warning(f"Invalid test type: {test[u'type']}")
 222                     continue
 223                 name = re.sub(REGEX_NIC, u"", test[u"parent"].
 224                               replace(u'-ndrpdr', u'').
 225                               replace(u'2n1l-', u'').
 226                               replace(u'avf-', u''))
 227                 histograms = list()
 228                 for idx_col, direction in enumerate(
 229                         (u"direction1", u"direction2", )):
 230                     try:
 231                         hdr_lat = test[u"latency"][ttype][direction][u"hdrh"]
 232                         # TODO: Workaround, HDRH data must be aligned to 4
 233                         #       bytes, remove when not needed.
 234                         hdr_lat += u"=" * (len(hdr_lat) % 4)
 235                         xaxis = list()
 236                         yaxis = list()
 237                         hovertext = list()
 238                         decoded = hdrh.histogram.HdrHistogram.decode(hdr_lat)
 239                         total_count = decoded.get_total_count()
 240                         for item in decoded.get_recorded_iterator():
 241                             xaxis.append(item.value_iterated_to)
 242                             prob = float(item.count_added_in_this_iter_step) / \
 243                                    total_count * 100
 244                             yaxis.append(prob)
 245                             hovertext.append(
 246                                 f"Test: {name}<br>"
 247                                 f"Direction: {directions[idx_col]}<br>"
 248                                 f"Latency: {item.value_iterated_to}uSec<br>"
 249                                 f"Probability: {prob:.2f}%<br>"
 250                                 f"Percentile: "
 251                                 f"{item.percentile_level_iterated_to:.2f}"
 252                             )
 253                         marker_color = [COLORS[idx_row], ] * len(yaxis)
 254                         marker_color[xaxis.index(
 255                             decoded.get_value_at_percentile(50.0))] = u"red"
 256                         marker_color[xaxis.index(
 257                             decoded.get_value_at_percentile(90.0))] = u"red"
 258                         marker_color[xaxis.index(
 259                             decoded.get_value_at_percentile(95.0))] = u"red"
 260                         histograms.append(
 261                             plgo.Bar(
 262                                 x=xaxis,
 263                                 y=yaxis,
 264                                 showlegend=False,
 265                                 name=name,
 266                                 marker={u"color": marker_color},
 267                                 hovertext=hovertext,
 268                                 hoverinfo=u"text"
 269                             )
 270                         )
 271                     except hdrh.codec.HdrLengthException as err:
 272                         logging.warning(
 273                             f"No or invalid data for HDRHistogram for the test "
 274                             f"{name}\n{err}"
 275                         )
 276                         continue
 277                 if len(histograms) == 2:
 278                     traces.append(histograms)
 279                     tests.append(name)
 280             else:
 281                 logging.warning(f"Invalid test type: {test[u'type']}")
 282                 continue
 283         except (ValueError, KeyError) as err:
 284             logging.warning(repr(err))
 285
 286     if not tests:
 287         logging.warning(f"No data for {plot_title}.")
 288         return
 289
 290     fig = make_subplots(
 291         rows=len(tests),
 292         cols=2,
 293         specs=[
 294             [{u"type": u"bar"}, {u"type": u"bar"}] for _ in range(len(tests))
 295         ]
 296     )
 297
 298     layout_axes = dict(
 299         gridcolor=u"rgb(220, 220, 220)",
 300         linecolor=u"rgb(220, 220, 220)",
 301         linewidth=1,
 302         showgrid=True,
 303         showline=True,
 304         showticklabels=True,
 305         tickcolor=u"rgb(220, 220, 220)",
 306     )
 307
 308     for idx_row, test in enumerate(tests):
 309         for idx_col in range(2):
 310             fig.add_trace(
 311                 traces[idx_row][idx_col],
 312                 row=idx_row + 1,
 313                 col=idx_col + 1
 314             )
 315             fig.update_xaxes(
 316                 row=idx_row + 1,
 317                 col=idx_col + 1,
 318                 **layout_axes
 319             )
 320             fig.update_yaxes(
 321                 row=idx_row + 1,
 322                 col=idx_col + 1,
 323                 **layout_axes
 324             )
 325
 326     layout = deepcopy(plot[u"layout"])
 327
 328     layout[u"title"][u"text"] = \
 329         f"<b>Latency:</b> {plot.get(u'graph-title', u'')}"
 330     layout[u"height"] = 250 * len(tests) + 130
 331
 332     layout[u"annotations"][2][u"y"] = 1.06 - 0.008 * len(tests)
 333     layout[u"annotations"][3][u"y"] = 1.06 - 0.008 * len(tests)
 334
 335     for idx, test in enumerate(tests):
 336         layout[u"annotations"].append({
 337             u"font": {
 338                 u"size": 14
 339             },
 340             u"showarrow": False,
 341             u"text": f"<b>{test}</b>",
 342             u"textangle": 0,
 343             u"x": 0.5,
 344             u"xanchor": u"center",
 345             u"xref": u"paper",
 346             u"y": 1.0 - float(idx) * 1.06 / len(tests),
 347             u"yanchor": u"bottom",
 348             u"yref": u"paper"
 349         })
 350
 351     fig[u"layout"].update(layout)
 352
 353     # Create plot
 354     file_type = plot.get(u"output-file-type", u".html")
 355     logging.info(f"    Writing file {plot[u'output-file']}{file_type}.")
 356     try:
 357         # Export Plot
 358         ploff.plot(fig, show_link=False, auto_open=False,
 359                    filename=f"{plot[u'output-file']}{file_type}")
 360     except PlotlyError as err:
 361         logging.error(f"   Finished with error: {repr(err)}")
 362
 363
 364 def plot_nf_reconf_box_name(plot, input_data):
 365     """Generate the plot(s) with algorithm: plot_nf_reconf_box_name
 366     specified in the specification file.
 367
 368     :param plot: Plot to generate.
 369     :param input_data: Data to process.
 370     :type plot: pandas.Series
 371     :type input_data: InputData
 372     """
 373
 374     # Transform the data
 375     logging.info(
 376         f"    Creating the data set for the {plot.get(u'type', u'')} "
 377         f"{plot.get(u'title', u'')}."
 378     )
 379     data = input_data.filter_tests_by_name(
 380         plot, params=[u"result", u"parent", u"tags", u"type"]
 381     )
 382     if data is None:
 383         logging.error(u"No data.")
 384         return
 385
 386     # Prepare the data for the plot
 387     y_vals = OrderedDict()
 388     loss = dict()
 389     for job in data:
 390         for build in job:
 391             for test in build:
 392                 if y_vals.get(test[u"parent"], None) is None:
 393                     y_vals[test[u"parent"]] = list()
 394                     loss[test[u"parent"]] = list()
 395                 try:
 396                     y_vals[test[u"parent"]].append(test[u"result"][u"time"])
 397                     loss[test[u"parent"]].append(test[u"result"][u"loss"])
 398                 except (KeyError, TypeError):
 399                     y_vals[test[u"parent"]].append(None)
 400
 401     # Add None to the lists with missing data
 402     max_len = 0
 403     nr_of_samples = list()
 404     for val in y_vals.values():
 405         if len(val) > max_len:
 406             max_len = len(val)
 407         nr_of_samples.append(len(val))
 408     for val in y_vals.values():
 409         if len(val) < max_len:
 410             val.extend([None for _ in range(max_len - len(val))])
 411
 412     # Add plot traces
 413     traces = list()
 414     df_y = pd.DataFrame(y_vals)
 415     df_y.head()
 416     for i, col in enumerate(df_y.columns):
 417         tst_name = re.sub(REGEX_NIC, u"",
 418                           col.lower().replace(u'-ndrpdr', u'').
 419                           replace(u'2n1l-', u'').replace(u'avf-', u''))
 420
 421         traces.append(plgo.Box(
 422             x=[str(i + 1) + u'.'] * len(df_y[col]),
 423             y=[y if y else None for y in df_y[col]],
 424             name=(
 425                 f"{i + 1}. "
 426                 f"({nr_of_samples[i]:02d} "
 427                 f"run{u's' if nr_of_samples[i] > 1 else u''}, "
 428                 f"packets lost average: {mean(loss[col]):.1f}) "
 429                 f"{u'-'.join(tst_name.split(u'-')[3:-2])}"
 430             ),
 431             hoverinfo=u"y+name"
 432         ))
 433     try:
 434         # Create plot
 435         layout = deepcopy(plot[u"layout"])
 436         layout[u"title"] = f"<b>Time Lost:</b> {layout[u'title']}"
 437         layout[u"yaxis"][u"title"] = u"<b>Implied Time Lost [s]</b>"
 438         layout[u"legend"][u"font"][u"size"] = 14
 439         layout[u"yaxis"].pop(u"range")
 440         plpl = plgo.Figure(data=traces, layout=layout)
 441
 442         # Export Plot
 443         file_type = plot.get(u"output-file-type", u".html")
 444         logging.info(f"    Writing file {plot[u'output-file']}{file_type}.")
 445         ploff.plot(
 446             plpl,
 447             show_link=False,
 448             auto_open=False,
 449             filename=f"{plot[u'output-file']}{file_type}"
 450         )
 451     except PlotlyError as err:
 452         logging.error(
 453             f"   Finished with error: {repr(err)}".replace(u"\n", u" ")
 454         )
 455         return
 456
 457
 458 def plot_perf_box_name(plot, input_data):
 459     """Generate the plot(s) with algorithm: plot_perf_box_name
 460     specified in the specification file.
 461
 462     :param plot: Plot to generate.
 463     :param input_data: Data to process.
 464     :type plot: pandas.Series
 465     :type input_data: InputData
 466     """
 467
 468     # Transform the data
 469     logging.info(
 470         f"    Creating data set for the {plot.get(u'type', u'')} "
 471         f"{plot.get(u'title', u'')}."
 472     )
 473     data = input_data.filter_tests_by_name(
 474         plot, params=[u"throughput", u"parent", u"tags", u"type"])
 475     if data is None:
 476         logging.error(u"No data.")
 477         return
 478
 479     # Prepare the data for the plot
 480     y_vals = OrderedDict()
 481     for job in data:
 482         for build in job:
 483             for test in build:
 484                 if y_vals.get(test[u"parent"], None) is None:
 485                     y_vals[test[u"parent"]] = list()
 486                 try:
 487                     if (test[u"type"] in (u"NDRPDR", ) and
 488                             u"-pdr" in plot.get(u"title", u"").lower()):
 489                         y_vals[test[u"parent"]].\
 490                             append(test[u"throughput"][u"PDR"][u"LOWER"])
 491                     elif (test[u"type"] in (u"NDRPDR", ) and
 492                           u"-ndr" in plot.get(u"title", u"").lower()):
 493                         y_vals[test[u"parent"]]. \
 494                             append(test[u"throughput"][u"NDR"][u"LOWER"])
 495                     elif test[u"type"] in (u"SOAK", ):
 496                         y_vals[test[u"parent"]].\
 497                             append(test[u"throughput"][u"LOWER"])
 498                     else:
 499                         continue
 500                 except (KeyError, TypeError):
 501                     y_vals[test[u"parent"]].append(None)
 502
 503     # Add None to the lists with missing data
 504     max_len = 0
 505     nr_of_samples = list()
 506     for val in y_vals.values():
 507         if len(val) > max_len:
 508             max_len = len(val)
 509         nr_of_samples.append(len(val))
 510     for val in y_vals.values():
 511         if len(val) < max_len:
 512             val.extend([None for _ in range(max_len - len(val))])
 513
 514     # Add plot traces
 515     traces = list()
 516     df_y = pd.DataFrame(y_vals)
 517     df_y.head()
 518     y_max = list()
 519     for i, col in enumerate(df_y.columns):
 520         tst_name = re.sub(REGEX_NIC, u"",
 521                           col.lower().replace(u'-ndrpdr', u'').
 522                           replace(u'2n1l-', u'').replace(u'avf-', u''))
 523         traces.append(
 524             plgo.Box(
 525                 x=[str(i + 1) + u'.'] * len(df_y[col]),
 526                 y=[y / 1000000 if y else None for y in df_y[col]],
 527                 name=(
 528                     f"{i + 1}. "
 529                     f"({nr_of_samples[i]:02d} "
 530                     f"run{u's' if nr_of_samples[i] > 1 else u''}) "
 531                     f"{tst_name}"
 532                 ),
 533                 hoverinfo=u"y+name"
 534             )
 535         )
 536         try:
 537             val_max = max(df_y[col])
 538             if val_max:
 539                 y_max.append(int(val_max / 1000000) + 2)
 540         except (ValueError, TypeError) as err:
 541             logging.error(repr(err))
 542             continue
 543
 544     try:
 545         # Create plot
 546         layout = deepcopy(plot[u"layout"])
 547         if layout.get(u"title", None):
 548             layout[u"title"] = f"<b>Throughput:</b> {layout[u'title']}"
 549         if y_max:
 550             layout[u"yaxis"][u"range"] = [0, max(y_max)]
 551         plpl = plgo.Figure(data=traces, layout=layout)
 552
 553         # Export Plot
 554         logging.info(f"    Writing file {plot[u'output-file']}.html.")
 555         ploff.plot(
 556             plpl,
 557             show_link=False,
 558             auto_open=False,
 559             filename=f"{plot[u'output-file']}.html"
 560         )
 561     except PlotlyError as err:
 562         logging.error(
 563             f"   Finished with error: {repr(err)}".replace(u"\n", u" ")
 564         )
 565         return
 566
 567
 568 def plot_lat_err_bars_name(plot, input_data):
 569     """Generate the plot(s) with algorithm: plot_lat_err_bars_name
 570     specified in the specification file.
 571
 572     :param plot: Plot to generate.
 573     :param input_data: Data to process.
 574     :type plot: pandas.Series
 575     :type input_data: InputData
 576     """
 577
 578     # Transform the data
 579     plot_title = plot.get(u"title", u"")
 580     logging.info(
 581         f"    Creating data set for the {plot.get(u'type', u'')} {plot_title}."
 582     )
 583     data = input_data.filter_tests_by_name(
 584         plot, params=[u"latency", u"parent", u"tags", u"type"])
 585     if data is None:
 586         logging.error(u"No data.")
 587         return
 588
 589     # Prepare the data for the plot
 590     y_tmp_vals = OrderedDict()
 591     for job in data:
 592         for build in job:
 593             for test in build:
 594                 try:
 595                     logging.debug(f"test[u'latency']: {test[u'latency']}\n")
 596                 except ValueError as err:
 597                     logging.warning(repr(err))
 598                 if y_tmp_vals.get(test[u"parent"], None) is None:
 599                     y_tmp_vals[test[u"parent"]] = [
 600                         list(),  # direction1, min
 601                         list(),  # direction1, avg
 602                         list(),  # direction1, max
 603                         list(),  # direction2, min
 604                         list(),  # direction2, avg
 605                         list()   # direction2, max
 606                     ]
 607                 try:
 608                     if test[u"type"] not in (u"NDRPDR", ):
 609                         logging.warning(f"Invalid test type: {test[u'type']}")
 610                         continue
 611                     if u"-pdr" in plot_title.lower():
 612                         ttype = u"PDR"
 613                     elif u"-ndr" in plot_title.lower():
 614                         ttype = u"NDR"
 615                     else:
 616                         logging.warning(
 617                             f"Invalid test type: {test[u'type']}"
 618                         )
 619                         continue
 620                     y_tmp_vals[test[u"parent"]][0].append(
 621                         test[u"latency"][ttype][u"direction1"][u"min"])
 622                     y_tmp_vals[test[u"parent"]][1].append(
 623                         test[u"latency"][ttype][u"direction1"][u"avg"])
 624                     y_tmp_vals[test[u"parent"]][2].append(
 625                         test[u"latency"][ttype][u"direction1"][u"max"])
 626                     y_tmp_vals[test[u"parent"]][3].append(
 627                         test[u"latency"][ttype][u"direction2"][u"min"])
 628                     y_tmp_vals[test[u"parent"]][4].append(
 629                         test[u"latency"][ttype][u"direction2"][u"avg"])
 630                     y_tmp_vals[test[u"parent"]][5].append(
 631                         test[u"latency"][ttype][u"direction2"][u"max"])
 632                 except (KeyError, TypeError) as err:
 633                     logging.warning(repr(err))
 634
 635     x_vals = list()
 636     y_vals = list()
 637     y_mins = list()
 638     y_maxs = list()
 639     nr_of_samples = list()
 640     for key, val in y_tmp_vals.items():
 641         name = re.sub(REGEX_NIC, u"", key.replace(u'-ndrpdr', u'').
 642                       replace(u'2n1l-', u'').replace(u'avf-', u''))
 643         x_vals.append(name)  # dir 1
 644         y_vals.append(mean(val[1]) if val[1] else None)
 645         y_mins.append(mean(val[0]) if val[0] else None)
 646         y_maxs.append(mean(val[2]) if val[2] else None)
 647         nr_of_samples.append(len(val[1]) if val[1] else 0)
 648         x_vals.append(name)  # dir 2
 649         y_vals.append(mean(val[4]) if val[4] else None)
 650         y_mins.append(mean(val[3]) if val[3] else None)
 651         y_maxs.append(mean(val[5]) if val[5] else None)
 652         nr_of_samples.append(len(val[3]) if val[3] else 0)
 653
 654     traces = list()
 655     annotations = list()
 656
 657     for idx, _ in enumerate(x_vals):
 658         if not bool(int(idx % 2)):
 659             direction = u"West-East"
 660         else:
 661             direction = u"East-West"
 662         hovertext = (
 663             f"No. of Runs: {nr_of_samples[idx]}<br>"
 664             f"Test: {x_vals[idx]}<br>"
 665             f"Direction: {direction}<br>"
 666         )
 667         if isinstance(y_maxs[idx], float):
 668             hovertext += f"Max: {y_maxs[idx]:.2f}uSec<br>"
 669         if isinstance(y_vals[idx], float):
 670             hovertext += f"Mean: {y_vals[idx]:.2f}uSec<br>"
 671         if isinstance(y_mins[idx], float):
 672             hovertext += f"Min: {y_mins[idx]:.2f}uSec"
 673
 674         if isinstance(y_maxs[idx], float) and isinstance(y_vals[idx], float):
 675             array = [y_maxs[idx] - y_vals[idx], ]
 676         else:
 677             array = [None, ]
 678         if isinstance(y_mins[idx], float) and isinstance(y_vals[idx], float):
 679             arrayminus = [y_vals[idx] - y_mins[idx], ]
 680         else:
 681             arrayminus = [None, ]
 682         traces.append(plgo.Scatter(
 683             x=[idx, ],
 684             y=[y_vals[idx], ],
 685             name=x_vals[idx],
 686             legendgroup=x_vals[idx],
 687             showlegend=bool(int(idx % 2)),
 688             mode=u"markers",
 689             error_y=dict(
 690                 type=u"data",
 691                 symmetric=False,
 692                 array=array,
 693                 arrayminus=arrayminus,
 694                 color=COLORS[int(idx / 2)]
 695             ),
 696             marker=dict(
 697                 size=10,
 698                 color=COLORS[int(idx / 2)],
 699             ),
 700             text=hovertext,
 701             hoverinfo=u"text",
 702         ))
 703         annotations.append(dict(
 704             x=idx,
 705             y=0,
 706             xref=u"x",
 707             yref=u"y",
 708             xanchor=u"center",
 709             yanchor=u"top",
 710             text=u"E-W" if bool(int(idx % 2)) else u"W-E",
 711             font=dict(
 712                 size=16,
 713             ),
 714             align=u"center",
 715             showarrow=False
 716         ))
 717
 718     try:
 719         # Create plot
 720         file_type = plot.get(u"output-file-type", u".html")
 721         logging.info(f"    Writing file {plot[u'output-file']}{file_type}.")
 722         layout = deepcopy(plot[u"layout"])
 723         if layout.get(u"title", None):
 724             layout[u"title"] = f"<b>Latency:</b> {layout[u'title']}"
 725         layout[u"annotations"] = annotations
 726         plpl = plgo.Figure(data=traces, layout=layout)
 727
 728         # Export Plot
 729         ploff.plot(
 730             plpl,
 731             show_link=False, auto_open=False,
 732             filename=f"{plot[u'output-file']}{file_type}"
 733         )
 734     except PlotlyError as err:
 735         logging.error(
 736             f"   Finished with error: {repr(err)}".replace(u"\n", u" ")
 737         )
 738         return
 739
 740
 741 def plot_tsa_name(plot, input_data):
 742     """Generate the plot(s) with algorithm:
 743     plot_tsa_name
 744     specified in the specification file.
 745
 746     :param plot: Plot to generate.
 747     :param input_data: Data to process.
 748     :type plot: pandas.Series
 749     :type input_data: InputData
 750     """
 751
 752     # Transform the data
 753     plot_title = plot.get(u"title", u"")
 754     logging.info(
 755         f"    Creating data set for the {plot.get(u'type', u'')} {plot_title}."
 756     )
 757     data = input_data.filter_tests_by_name(
 758         plot, params=[u"throughput", u"parent", u"tags", u"type"])
 759     if data is None:
 760         logging.error(u"No data.")
 761         return
 762
 763     y_vals = OrderedDict()
 764     for job in data:
 765         for build in job:
 766             for test in build:
 767                 if y_vals.get(test[u"parent"], None) is None:
 768                     y_vals[test[u"parent"]] = {
 769                         u"1": list(),
 770                         u"2": list(),
 771                         u"4": list()
 772                     }
 773                 try:
 774                     if test[u"type"] not in (u"NDRPDR",):
 775                         continue
 776
 777                     if u"-pdr" in plot_title.lower():
 778                         ttype = u"PDR"
 779                     elif u"-ndr" in plot_title.lower():
 780                         ttype = u"NDR"
 781                     else:
 782                         continue
 783
 784                     if u"1C" in test[u"tags"]:
 785                         y_vals[test[u"parent"]][u"1"]. \
 786                             append(test[u"throughput"][ttype][u"LOWER"])
 787                     elif u"2C" in test[u"tags"]:
 788                         y_vals[test[u"parent"]][u"2"]. \
 789                             append(test[u"throughput"][ttype][u"LOWER"])
 790                     elif u"4C" in test[u"tags"]:
 791                         y_vals[test[u"parent"]][u"4"]. \
 792                             append(test[u"throughput"][ttype][u"LOWER"])
 793                 except (KeyError, TypeError):
 794                     pass
 795
 796     if not y_vals:
 797         logging.warning(f"No data for the plot {plot.get(u'title', u'')}")
 798         return
 799
 800     y_1c_max = dict()
 801     for test_name, test_vals in y_vals.items():
 802         for key, test_val in test_vals.items():
 803             if test_val:
 804                 avg_val = sum(test_val) / len(test_val)
 805                 y_vals[test_name][key] = [avg_val, len(test_val)]
 806                 ideal = avg_val / (int(key) * 1000000.0)
 807                 if test_name not in y_1c_max or ideal > y_1c_max[test_name]:
 808                     y_1c_max[test_name] = ideal
 809
 810     vals = OrderedDict()
 811     y_max = list()
 812     nic_limit = 0
 813     lnk_limit = 0
 814     pci_limit = plot[u"limits"][u"pci"][u"pci-g3-x8"]
 815     for test_name, test_vals in y_vals.items():
 816         try:
 817             if test_vals[u"1"][1]:
 818                 name = re.sub(
 819                     REGEX_NIC,
 820                     u"",
 821                     test_name.replace(u'-ndrpdr', u'').replace(u'2n1l-', u'').
 822                         replace(u'avf-', u'')
 823                 )
 824                 vals[name] = OrderedDict()
 825                 y_val_1 = test_vals[u"1"][0] / 1000000.0
 826                 y_val_2 = test_vals[u"2"][0] / 1000000.0 if test_vals[u"2"][0] \
 827                     else None
 828                 y_val_4 = test_vals[u"4"][0] / 1000000.0 if test_vals[u"4"][0] \
 829                     else None
 830
 831                 vals[name][u"val"] = [y_val_1, y_val_2, y_val_4]
 832                 vals[name][u"rel"] = [1.0, None, None]
 833                 vals[name][u"ideal"] = [
 834                     y_1c_max[test_name],
 835                     y_1c_max[test_name] * 2,
 836                     y_1c_max[test_name] * 4
 837                 ]
 838                 vals[name][u"diff"] = [
 839                     (y_val_1 - y_1c_max[test_name]) * 100 / y_val_1, None, None
 840                 ]
 841                 vals[name][u"count"] = [
 842                     test_vals[u"1"][1],
 843                     test_vals[u"2"][1],
 844                     test_vals[u"4"][1]
 845                 ]
 846
 847                 try:
 848                     val_max = max(vals[name][u"val"])
 849                 except ValueError as err:
 850                     logging.error(repr(err))
 851                     continue
 852                 if val_max:
 853                     y_max.append(val_max)
 854
 855                 if y_val_2:
 856                     vals[name][u"rel"][1] = round(y_val_2 / y_val_1, 2)
 857                     vals[name][u"diff"][1] = \
 858                         (y_val_2 - vals[name][u"ideal"][1]) * 100 / y_val_2
 859                 if y_val_4:
 860                     vals[name][u"rel"][2] = round(y_val_4 / y_val_1, 2)
 861                     vals[name][u"diff"][2] = \
 862                         (y_val_4 - vals[name][u"ideal"][2]) * 100 / y_val_4
 863         except IndexError as err:
 864             logging.warning(f"No data for {test_name}")
 865             logging.warning(repr(err))
 866
 867         # Limits:
 868         if u"x520" in test_name:
 869             limit = plot[u"limits"][u"nic"][u"x520"]
 870         elif u"x710" in test_name:
 871             limit = plot[u"limits"][u"nic"][u"x710"]
 872         elif u"xxv710" in test_name:
 873             limit = plot[u"limits"][u"nic"][u"xxv710"]
 874         elif u"xl710" in test_name:
 875             limit = plot[u"limits"][u"nic"][u"xl710"]
 876         elif u"x553" in test_name:
 877             limit = plot[u"limits"][u"nic"][u"x553"]
 878         else:
 879             limit = 0
 880         if limit > nic_limit:
 881             nic_limit = limit
 882
 883         mul = 2 if u"ge2p" in test_name else 1
 884         if u"10ge" in test_name:
 885             limit = plot[u"limits"][u"link"][u"10ge"] * mul
 886         elif u"25ge" in test_name:
 887             limit = plot[u"limits"][u"link"][u"25ge"] * mul
 888         elif u"40ge" in test_name:
 889             limit = plot[u"limits"][u"link"][u"40ge"] * mul
 890         elif u"100ge" in test_name:
 891             limit = plot[u"limits"][u"link"][u"100ge"] * mul
 892         else:
 893             limit = 0
 894         if limit > lnk_limit:
 895             lnk_limit = limit
 896
 897     traces = list()
 898     annotations = list()
 899     x_vals = [1, 2, 4]
 900
 901     # Limits:
 902     try:
 903         threshold = 1.1 * max(y_max)  # 10%
 904     except ValueError as err:
 905         logging.error(err)
 906         return
 907     nic_limit /= 1000000.0
 908     traces.append(plgo.Scatter(
 909         x=x_vals,
 910         y=[nic_limit, ] * len(x_vals),
 911         name=f"NIC: {nic_limit:.2f}Mpps",
 912         showlegend=False,
 913         mode=u"lines",
 914         line=dict(
 915             dash=u"dot",
 916             color=COLORS[-1],
 917             width=1),
 918         hoverinfo=u"none"
 919     ))
 920     annotations.append(dict(
 921         x=1,
 922         y=nic_limit,
 923         xref=u"x",
 924         yref=u"y",
 925         xanchor=u"left",
 926         yanchor=u"bottom",
 927         text=f"NIC: {nic_limit:.2f}Mpps",
 928         font=dict(
 929             size=14,
 930             color=COLORS[-1],
 931         ),
 932         align=u"left",
 933         showarrow=False
 934     ))
 935     y_max.append(nic_limit)
 936
 937     lnk_limit /= 1000000.0
 938     if lnk_limit < threshold:
 939         traces.append(plgo.Scatter(
 940             x=x_vals,
 941             y=[lnk_limit, ] * len(x_vals),
 942             name=f"Link: {lnk_limit:.2f}Mpps",
 943             showlegend=False,
 944             mode=u"lines",
 945             line=dict(
 946                 dash=u"dot",
 947                 color=COLORS[-2],
 948                 width=1),
 949             hoverinfo=u"none"
 950         ))
 951         annotations.append(dict(
 952             x=1,
 953             y=lnk_limit,
 954             xref=u"x",
 955             yref=u"y",
 956             xanchor=u"left",
 957             yanchor=u"bottom",
 958             text=f"Link: {lnk_limit:.2f}Mpps",
 959             font=dict(
 960                 size=14,
 961                 color=COLORS[-2],
 962             ),
 963             align=u"left",
 964             showarrow=False
 965         ))
 966         y_max.append(lnk_limit)
 967
 968     pci_limit /= 1000000.0
 969     if (pci_limit < threshold and
 970             (pci_limit < lnk_limit * 0.95 or lnk_limit > lnk_limit * 1.05)):
 971         traces.append(plgo.Scatter(
 972             x=x_vals,
 973             y=[pci_limit, ] * len(x_vals),
 974             name=f"PCIe: {pci_limit:.2f}Mpps",
 975             showlegend=False,
 976             mode=u"lines",
 977             line=dict(
 978                 dash=u"dot",
 979                 color=COLORS[-3],
 980                 width=1),
 981             hoverinfo=u"none"
 982         ))
 983         annotations.append(dict(
 984             x=1,
 985             y=pci_limit,
 986             xref=u"x",
 987             yref=u"y",
 988             xanchor=u"left",
 989             yanchor=u"bottom",
 990             text=f"PCIe: {pci_limit:.2f}Mpps",
 991             font=dict(
 992                 size=14,
 993                 color=COLORS[-3],
 994             ),
 995             align=u"left",
 996             showarrow=False
 997         ))
 998         y_max.append(pci_limit)
 999
1000     # Perfect and measured:
1001     cidx = 0
1002     for name, val in vals.items():
1003         hovertext = list()
1004         try:
1005             for idx in range(len(val[u"val"])):
1006                 htext = ""
1007                 if isinstance(val[u"val"][idx], float):
1008                     htext += (
1009                         f"No. of Runs: {val[u'count'][idx]}<br>"
1010                         f"Mean: {val[u'val'][idx]:.2f}Mpps<br>"
1011                     )
1012                 if isinstance(val[u"diff"][idx], float):
1013                     htext += f"Diff: {round(val[u'diff'][idx]):.0f}%<br>"
1014                 if isinstance(val[u"rel"][idx], float):
1015                     htext += f"Speedup: {val[u'rel'][idx]:.2f}"
1016                 hovertext.append(htext)
1017             traces.append(
1018                 plgo.Scatter(
1019                     x=x_vals,
1020                     y=val[u"val"],
1021                     name=name,
1022                     legendgroup=name,
1023                     mode=u"lines+markers",
1024                     line=dict(
1025                         color=COLORS[cidx],
1026                         width=2),
1027                     marker=dict(
1028                         symbol=u"circle",
1029                         size=10
1030                     ),
1031                     text=hovertext,
1032                     hoverinfo=u"text+name"
1033                 )
1034             )
1035             traces.append(
1036                 plgo.Scatter(
1037                     x=x_vals,
1038                     y=val[u"ideal"],
1039                     name=f"{name} perfect",
1040                     legendgroup=name,
1041                     showlegend=False,
1042                     mode=u"lines",
1043                     line=dict(
1044                         color=COLORS[cidx],
1045                         width=2,
1046                         dash=u"dash"),
1047                     text=[f"Perfect: {y:.2f}Mpps" for y in val[u"ideal"]],
1048                     hoverinfo=u"text"
1049                 )
1050             )
1051             cidx += 1
1052         except (IndexError, ValueError, KeyError) as err:
1053             logging.warning(f"No data for {name}\n{repr(err)}")
1054
1055     try:
1056         # Create plot
1057         file_type = plot.get(u"output-file-type", u".html")
1058         logging.info(f"    Writing file {plot[u'output-file']}{file_type}.")
1059         layout = deepcopy(plot[u"layout"])
1060         if layout.get(u"title", None):
1061             layout[u"title"] = f"<b>Speedup Multi-core:</b> {layout[u'title']}"
1062         layout[u"yaxis"][u"range"] = [0, int(max(y_max) * 1.1)]
1063         layout[u"annotations"].extend(annotations)
1064         plpl = plgo.Figure(data=traces, layout=layout)
1065
1066         # Export Plot
1067         ploff.plot(
1068             plpl,
1069             show_link=False,
1070             auto_open=False,
1071             filename=f"{plot[u'output-file']}{file_type}"
1072         )
1073     except PlotlyError as err:
1074         logging.error(
1075             f"   Finished with error: {repr(err)}".replace(u"\n", u" ")
1076         )
1077         return
1078
1079
1080 def plot_http_server_perf_box(plot, input_data):
1081     """Generate the plot(s) with algorithm: plot_http_server_perf_box
1082     specified in the specification file.
1083
1084     :param plot: Plot to generate.
1085     :param input_data: Data to process.
1086     :type plot: pandas.Series
1087     :type input_data: InputData
1088     """
1089
1090     # Transform the data
1091     logging.info(
1092         f"    Creating the data set for the {plot.get(u'type', u'')} "
1093         f"{plot.get(u'title', u'')}."
1094     )
1095     data = input_data.filter_data(plot)
1096     if data is None:
1097         logging.error(u"No data.")
1098         return
1099
1100     # Prepare the data for the plot
1101     y_vals = dict()
1102     for job in data:
1103         for build in job:
1104             for test in build:
1105                 if y_vals.get(test[u"name"], None) is None:
1106                     y_vals[test[u"name"]] = list()
1107                 try:
1108                     y_vals[test[u"name"]].append(test[u"result"])
1109                 except (KeyError, TypeError):
1110                     y_vals[test[u"name"]].append(None)
1111
1112     # Add None to the lists with missing data
1113     max_len = 0
1114     nr_of_samples = list()
1115     for val in y_vals.values():
1116         if len(val) > max_len:
1117             max_len = len(val)
1118         nr_of_samples.append(len(val))
1119     for val in y_vals.values():
1120         if len(val) < max_len:
1121             val.extend([None for _ in range(max_len - len(val))])
1122
1123     # Add plot traces
1124     traces = list()
1125     df_y = pd.DataFrame(y_vals)
1126     df_y.head()
1127     for i, col in enumerate(df_y.columns):
1128         name = \
1129             f"{i + 1}. " \
1130             f"({nr_of_samples[i]:02d} " \
1131             f"run{u's' if nr_of_samples[i] > 1 else u''}) " \
1132             f"{col.lower().replace(u'-ndrpdr', u'')}"
1133         if len(name) > 50:
1134             name_lst = name.split(u'-')
1135             name = u""
1136             split_name = True
1137             for segment in name_lst:
1138                 if (len(name) + len(segment) + 1) > 50 and split_name:
1139                     name += u"<br>    "
1140                     split_name = False
1141                 name += segment + u'-'
1142             name = name[:-1]
1143
1144         traces.append(plgo.Box(x=[str(i + 1) + u'.'] * len(df_y[col]),
1145                                y=df_y[col],
1146                                name=name,
1147                                **plot[u"traces"]))
1148     try:
1149         # Create plot
1150         plpl = plgo.Figure(data=traces, layout=plot[u"layout"])
1151
1152         # Export Plot
1153         logging.info(
1154             f"    Writing file {plot[u'output-file']}"
1155             f"{plot[u'output-file-type']}."
1156         )
1157         ploff.plot(
1158             plpl,
1159             show_link=False,
1160             auto_open=False,
1161             filename=f"{plot[u'output-file']}{plot[u'output-file-type']}"
1162         )
1163     except PlotlyError as err:
1164         logging.error(
1165             f"   Finished with error: {repr(err)}".replace(u"\n", u" ")
1166         )
1167         return
1168
1169
1170 def plot_nf_heatmap(plot, input_data):
1171     """Generate the plot(s) with algorithm: plot_nf_heatmap
1172     specified in the specification file.
1173
1174     :param plot: Plot to generate.
1175     :param input_data: Data to process.
1176     :type plot: pandas.Series
1177     :type input_data: InputData
1178     """
1179
1180     regex_cn = re.compile(r'^(\d*)R(\d*)C$')
1181     regex_test_name = re.compile(r'^.*-(\d+ch|\d+pl)-'
1182                                  r'(\d+mif|\d+vh)-'
1183                                  r'(\d+vm\d+t|\d+dcr\d+t).*$')
1184     vals = dict()
1185
1186     # Transform the data
1187     logging.info(
1188         f"    Creating the data set for the {plot.get(u'type', u'')} "
1189         f"{plot.get(u'title', u'')}."
1190     )
1191     data = input_data.filter_data(plot, continue_on_error=True)
1192     if data is None or data.empty:
1193         logging.error(u"No data.")
1194         return
1195
1196     for job in data:
1197         for build in job:
1198             for test in build:
1199                 for tag in test[u"tags"]:
1200                     groups = re.search(regex_cn, tag)
1201                     if groups:
1202                         chain = str(groups.group(1))
1203                         node = str(groups.group(2))
1204                         break
1205                 else:
1206                     continue
1207                 groups = re.search(regex_test_name, test[u"name"])
1208                 if groups and len(groups.groups()) == 3:
1209                     hover_name = (
1210                         f"{str(groups.group(1))}-"
1211                         f"{str(groups.group(2))}-"
1212                         f"{str(groups.group(3))}"
1213                     )
1214                 else:
1215                     hover_name = u""
1216                 if vals.get(chain, None) is None:
1217                     vals[chain] = dict()
1218                 if vals[chain].get(node, None) is None:
1219                     vals[chain][node] = dict(
1220                         name=hover_name,
1221                         vals=list(),
1222                         nr=None,
1223                         mean=None,
1224                         stdev=None
1225                     )
1226                 try:
1227                     if plot[u"include-tests"] == u"MRR":
1228                         result = test[u"result"][u"receive-rate"]
1229                     elif plot[u"include-tests"] == u"PDR":
1230                         result = test[u"throughput"][u"PDR"][u"LOWER"]
1231                     elif plot[u"include-tests"] == u"NDR":
1232                         result = test[u"throughput"][u"NDR"][u"LOWER"]
1233                     else:
1234                         result = None
1235                 except TypeError:
1236                     result = None
1237
1238                 if result:
1239                     vals[chain][node][u"vals"].append(result)
1240
1241     if not vals:
1242         logging.error(u"No data.")
1243         return
1244
1245     txt_chains = list()
1246     txt_nodes = list()
1247     for key_c in vals:
1248         txt_chains.append(key_c)
1249         for key_n in vals[key_c].keys():
1250             txt_nodes.append(key_n)
1251             if vals[key_c][key_n][u"vals"]:
1252                 vals[key_c][key_n][u"nr"] = len(vals[key_c][key_n][u"vals"])
1253                 vals[key_c][key_n][u"mean"] = \
1254                     round(mean(vals[key_c][key_n][u"vals"]) / 1000000, 1)
1255                 vals[key_c][key_n][u"stdev"] = \
1256                     round(stdev(vals[key_c][key_n][u"vals"]) / 1000000, 1)
1257     txt_nodes = list(set(txt_nodes))
1258
1259     def sort_by_int(value):
1260         """Makes possible to sort a list of strings which represent integers.
1261
1262         :param value: Integer as a string.
1263         :type value: str
1264         :returns: Integer representation of input parameter 'value'.
1265         :rtype: int
1266         """
1267         return int(value)
1268
1269     txt_chains = sorted(txt_chains, key=sort_by_int)
1270     txt_nodes = sorted(txt_nodes, key=sort_by_int)
1271
1272     chains = [i + 1 for i in range(len(txt_chains))]
1273     nodes = [i + 1 for i in range(len(txt_nodes))]
1274
1275     data = [list() for _ in range(len(chains))]
1276     for chain in chains:
1277         for node in nodes:
1278             try:
1279                 val = vals[txt_chains[chain - 1]][txt_nodes[node - 1]][u"mean"]
1280             except (KeyError, IndexError):
1281                 val = None
1282             data[chain - 1].append(val)
1283
1284     # Color scales:
1285     my_green = [[0.0, u"rgb(235, 249, 242)"],
1286                 [1.0, u"rgb(45, 134, 89)"]]
1287
1288     my_blue = [[0.0, u"rgb(236, 242, 248)"],
1289                [1.0, u"rgb(57, 115, 172)"]]
1290
1291     my_grey = [[0.0, u"rgb(230, 230, 230)"],
1292                [1.0, u"rgb(102, 102, 102)"]]
1293
1294     hovertext = list()
1295     annotations = list()
1296
1297     text = (u"Test: {name}<br>"
1298             u"Runs: {nr}<br>"
1299             u"Thput: {val}<br>"
1300             u"StDev: {stdev}")
1301
1302     for chain, _ in enumerate(txt_chains):
1303         hover_line = list()
1304         for node, _ in enumerate(txt_nodes):
1305             if data[chain][node] is not None:
1306                 annotations.append(
1307                     dict(
1308                         x=node+1,
1309                         y=chain+1,
1310                         xref=u"x",
1311                         yref=u"y",
1312                         xanchor=u"center",
1313                         yanchor=u"middle",
1314                         text=str(data[chain][node]),
1315                         font=dict(
1316                             size=14,
1317                         ),
1318                         align=u"center",
1319                         showarrow=False
1320                     )
1321                 )
1322                 hover_line.append(text.format(
1323                     name=vals[txt_chains[chain]][txt_nodes[node]][u"name"],
1324                     nr=vals[txt_chains[chain]][txt_nodes[node]][u"nr"],
1325                     val=data[chain][node],
1326                     stdev=vals[txt_chains[chain]][txt_nodes[node]][u"stdev"]))
1327         hovertext.append(hover_line)
1328
1329     traces = [
1330         plgo.Heatmap(
1331             x=nodes,
1332             y=chains,
1333             z=data,
1334             colorbar=dict(
1335                 title=plot.get(u"z-axis", u""),
1336                 titleside=u"right",
1337                 titlefont=dict(
1338                     size=16
1339                 ),
1340                 tickfont=dict(
1341                     size=16,
1342                 ),
1343                 tickformat=u".1f",
1344                 yanchor=u"bottom",
1345                 y=-0.02,
1346                 len=0.925,
1347             ),
1348             showscale=True,
1349             colorscale=my_green,
1350             text=hovertext,
1351             hoverinfo=u"text"
1352         )
1353     ]
1354
1355     for idx, item in enumerate(txt_nodes):
1356         # X-axis, numbers:
1357         annotations.append(
1358             dict(
1359                 x=idx+1,
1360                 y=0.05,
1361                 xref=u"x",
1362                 yref=u"y",
1363                 xanchor=u"center",
1364                 yanchor=u"top",
1365                 text=item,
1366                 font=dict(
1367                     size=16,
1368                 ),
1369                 align=u"center",
1370                 showarrow=False
1371             )
1372         )
1373     for idx, item in enumerate(txt_chains):
1374         # Y-axis, numbers:
1375         annotations.append(
1376             dict(
1377                 x=0.35,
1378                 y=idx+1,
1379                 xref=u"x",
1380                 yref=u"y",
1381                 xanchor=u"right",
1382                 yanchor=u"middle",
1383                 text=item,
1384                 font=dict(
1385                     size=16,
1386                 ),
1387                 align=u"center",
1388                 showarrow=False
1389             )
1390         )
1391     # X-axis, title:
1392     annotations.append(
1393         dict(
1394             x=0.55,
1395             y=-0.15,
1396             xref=u"paper",
1397             yref=u"y",
1398             xanchor=u"center",
1399             yanchor=u"bottom",
1400             text=plot.get(u"x-axis", u""),
1401             font=dict(
1402                 size=16,
1403             ),
1404             align=u"center",
1405             showarrow=False
1406         )
1407     )
1408     # Y-axis, title:
1409     annotations.append(
1410         dict(
1411             x=-0.1,
1412             y=0.5,
1413             xref=u"x",
1414             yref=u"paper",
1415             xanchor=u"center",
1416             yanchor=u"middle",
1417             text=plot.get(u"y-axis", u""),
1418             font=dict(
1419                 size=16,
1420             ),
1421             align=u"center",
1422             textangle=270,
1423             showarrow=False
1424         )
1425     )
1426     updatemenus = list([
1427         dict(
1428             x=1.0,
1429             y=0.0,
1430             xanchor=u"right",
1431             yanchor=u"bottom",
1432             direction=u"up",
1433             buttons=list([
1434                 dict(
1435                     args=[
1436                         {
1437                             u"colorscale": [my_green, ],
1438                             u"reversescale": False
1439                         }
1440                     ],
1441                     label=u"Green",
1442                     method=u"update"
1443                 ),
1444                 dict(
1445                     args=[
1446                         {
1447                             u"colorscale": [my_blue, ],
1448                             u"reversescale": False
1449                         }
1450                     ],
1451                     label=u"Blue",
1452                     method=u"update"
1453                 ),
1454                 dict(
1455                     args=[
1456                         {
1457                             u"colorscale": [my_grey, ],
1458                             u"reversescale": False
1459                         }
1460                     ],
1461                     label=u"Grey",
1462                     method=u"update"
1463                 )
1464             ])
1465         )
1466     ])
1467
1468     try:
1469         layout = deepcopy(plot[u"layout"])
1470     except KeyError as err:
1471         logging.error(f"Finished with error: No layout defined\n{repr(err)}")
1472         return
1473
1474     layout[u"annotations"] = annotations
1475     layout[u'updatemenus'] = updatemenus
1476
1477     try:
1478         # Create plot
1479         plpl = plgo.Figure(data=traces, layout=layout)
1480
1481         # Export Plot
1482         logging.info(
1483             f"    Writing file {plot[u'output-file']}"
1484             f"{plot[u'output-file-type']}."
1485         )
1486         ploff.plot(
1487             plpl,
1488             show_link=False,
1489             auto_open=False,
1490             filename=f"{plot[u'output-file']}{plot[u'output-file-type']}"
1491         )
1492     except PlotlyError as err:
1493         logging.error(
1494             f"   Finished with error: {repr(err)}".replace(u"\n", u" ")
1495         )
1496         return