resources/tools/presentation/generator_plots.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate plots.
  15 """
  16
  17
  18 import re
  19 import logging
  20 import pandas as pd
  21 import plotly.offline as ploff
  22 import plotly.graph_objs as plgo
  23
  24 from plotly.exceptions import PlotlyError
  25 from collections import OrderedDict
  26 from copy import deepcopy
  27
  28 from utils import mean, stdev
  29
  30
  31 COLORS = ["SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
  32           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
  33           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
  34           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
  35           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
  36           "MediumSeaGreen", "SeaGreen", "LightSlateGrey"]
  37
  38 REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*-')
  39
  40
  41 def generate_plots(spec, data):
  42     """Generate all plots specified in the specification file.
  43
  44     :param spec: Specification read from the specification file.
  45     :param data: Data to process.
  46     :type spec: Specification
  47     :type data: InputData
  48     """
  49
  50     logging.info("Generating the plots ...")
  51     for index, plot in enumerate(spec.plots):
  52         try:
  53             logging.info("  Plot nr {0}: {1}".format(index + 1,
  54                                                      plot.get("title", "")))
  55             plot["limits"] = spec.configuration["limits"]
  56             eval(plot["algorithm"])(plot, data)
  57             logging.info("  Done.")
  58         except NameError as err:
  59             logging.error("Probably algorithm '{alg}' is not defined: {err}".
  60                           format(alg=plot["algorithm"], err=repr(err)))
  61     logging.info("Done.")
  62
  63
  64 def plot_performance_box(plot, input_data):
  65     """Generate the plot(s) with algorithm: plot_performance_box
  66     specified in the specification file.
  67
  68     :param plot: Plot to generate.
  69     :param input_data: Data to process.
  70     :type plot: pandas.Series
  71     :type input_data: InputData
  72     """
  73
  74     # Transform the data
  75     plot_title = plot.get("title", "")
  76     logging.info("    Creating the data set for the {0} '{1}'.".
  77                  format(plot.get("type", ""), plot_title))
  78     data = input_data.filter_data(plot)
  79     if data is None:
  80         logging.error("No data.")
  81         return
  82
  83     # Prepare the data for the plot
  84     y_vals = dict()
  85     y_tags = dict()
  86     for job in data:
  87         for build in job:
  88             for test in build:
  89                 if y_vals.get(test["parent"], None) is None:
  90                     y_vals[test["parent"]] = list()
  91                     y_tags[test["parent"]] = test.get("tags", None)
  92                 try:
  93                     if test["type"] in ("NDRPDR", ):
  94                         if "-pdr" in plot_title.lower():
  95                             y_vals[test["parent"]].\
  96                                 append(test["throughput"]["PDR"]["LOWER"])
  97                         elif "-ndr" in plot_title.lower():
  98                             y_vals[test["parent"]]. \
  99                                 append(test["throughput"]["NDR"]["LOWER"])
 100                         else:
 101                             continue
 102                     else:
 103                         continue
 104                 except (KeyError, TypeError):
 105                     y_vals[test["parent"]].append(None)
 106
 107     # Sort the tests
 108     order = plot.get("sort", None)
 109     if order and y_tags:
 110         y_sorted = OrderedDict()
 111         y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()}
 112         for tag in order:
 113             logging.debug(tag)
 114             for suite, tags in y_tags_l.items():
 115                 if "not " in tag:
 116                     tag = tag.split(" ")[-1]
 117                     if tag.lower() in tags:
 118                         continue
 119                 else:
 120                     if tag.lower() not in tags:
 121                         continue
 122                 try:
 123                     y_sorted[suite] = y_vals.pop(suite)
 124                     y_tags_l.pop(suite)
 125                     logging.debug(suite)
 126                 except KeyError as err:
 127                     logging.error("Not found: {0}".format(repr(err)))
 128                 finally:
 129                     break
 130     else:
 131         y_sorted = y_vals
 132
 133     # Add None to the lists with missing data
 134     max_len = 0
 135     nr_of_samples = list()
 136     for val in y_sorted.values():
 137         if len(val) > max_len:
 138             max_len = len(val)
 139         nr_of_samples.append(len(val))
 140     for key, val in y_sorted.items():
 141         if len(val) < max_len:
 142             val.extend([None for _ in range(max_len - len(val))])
 143
 144     # Add plot traces
 145     traces = list()
 146     df = pd.DataFrame(y_sorted)
 147     df.head()
 148     y_max = list()
 149     for i, col in enumerate(df.columns):
 150         tst_name = re.sub(REGEX_NIC, "",
 151                           col.lower().replace('-ndrpdr', '').
 152                           replace('2n1l-', ''))
 153         name = "{nr}. ({samples:02d} run{plural}) {name}".\
 154             format(nr=(i + 1),
 155                    samples=nr_of_samples[i],
 156                    plural='s' if nr_of_samples[i] > 1 else '',
 157                    name=tst_name)
 158
 159         logging.debug(name)
 160         traces.append(plgo.Box(x=[str(i + 1) + '.'] * len(df[col]),
 161                                y=[y / 1000000 if y else None for y in df[col]],
 162                                name=name,
 163                                **plot["traces"]))
 164         try:
 165             val_max = max(df[col])
 166         except ValueError as err:
 167             logging.error(repr(err))
 168             continue
 169         if val_max:
 170             y_max.append(int(val_max / 1000000) + 1)
 171
 172     try:
 173         # Create plot
 174         layout = deepcopy(plot["layout"])
 175         if layout.get("title", None):
 176             layout["title"] = "<b>Throughput:</b> {0}". \
 177                 format(layout["title"])
 178         if y_max:
 179             layout["yaxis"]["range"] = [0, max(y_max)]
 180         plpl = plgo.Figure(data=traces, layout=layout)
 181
 182         # Export Plot
 183         logging.info("    Writing file '{0}{1}'.".
 184                      format(plot["output-file"], plot["output-file-type"]))
 185         ploff.plot(plpl, show_link=False, auto_open=False,
 186                    filename='{0}{1}'.format(plot["output-file"],
 187                                             plot["output-file-type"]))
 188     except PlotlyError as err:
 189         logging.error("   Finished with error: {}".
 190                       format(repr(err).replace("\n", " ")))
 191         return
 192
 193
 194 def plot_soak_bars(plot, input_data):
 195     """Generate the plot(s) with algorithm: plot_soak_bars
 196     specified in the specification file.
 197
 198     :param plot: Plot to generate.
 199     :param input_data: Data to process.
 200     :type plot: pandas.Series
 201     :type input_data: InputData
 202     """
 203
 204     # Transform the data
 205     plot_title = plot.get("title", "")
 206     logging.info("    Creating the data set for the {0} '{1}'.".
 207                  format(plot.get("type", ""), plot_title))
 208     data = input_data.filter_data(plot)
 209     if data is None:
 210         logging.error("No data.")
 211         return
 212
 213     # Prepare the data for the plot
 214     y_vals = dict()
 215     y_tags = dict()
 216     for job in data:
 217         for build in job:
 218             for test in build:
 219                 if y_vals.get(test["parent"], None) is None:
 220                     y_tags[test["parent"]] = test.get("tags", None)
 221                 try:
 222                     if test["type"] in ("SOAK", ):
 223                         y_vals[test["parent"]] = test["throughput"]
 224                     else:
 225                         continue
 226                 except (KeyError, TypeError):
 227                     y_vals[test["parent"]] = dict()
 228
 229     # Sort the tests
 230     order = plot.get("sort", None)
 231     if order and y_tags:
 232         y_sorted = OrderedDict()
 233         y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()}
 234         for tag in order:
 235             logging.debug(tag)
 236             for suite, tags in y_tags_l.items():
 237                 if "not " in tag:
 238                     tag = tag.split(" ")[-1]
 239                     if tag.lower() in tags:
 240                         continue
 241                 else:
 242                     if tag.lower() not in tags:
 243                         continue
 244                 try:
 245                     y_sorted[suite] = y_vals.pop(suite)
 246                     y_tags_l.pop(suite)
 247                     logging.debug(suite)
 248                 except KeyError as err:
 249                     logging.error("Not found: {0}".format(repr(err)))
 250                 finally:
 251                     break
 252     else:
 253         y_sorted = y_vals
 254
 255     idx = 0
 256     y_max = 0
 257     traces = list()
 258     for test_name, test_data in y_sorted.items():
 259         idx += 1
 260         name = "{nr}. {name}".\
 261             format(nr=idx, name=test_name.lower().replace('-soak', ''))
 262         if len(name) > 50:
 263             name_lst = name.split('-')
 264             name = ""
 265             split_name = True
 266             for segment in name_lst:
 267                 if (len(name) + len(segment) + 1) > 50 and split_name:
 268                     name += "<br>    "
 269                     split_name = False
 270                 name += segment + '-'
 271             name = name[:-1]
 272
 273         y_val = test_data.get("LOWER", None)
 274         if y_val:
 275             y_val /= 1000000
 276             if y_val > y_max:
 277                 y_max = y_val
 278
 279         time = "No Information"
 280         result = "No Information"
 281         hovertext = ("{name}<br>"
 282                      "Packet Throughput: {val:.2f}Mpps<br>"
 283                      "Final Duration: {time}<br>"
 284                      "Result: {result}".format(name=name,
 285                                                val=y_val,
 286                                                time=time,
 287                                                result=result))
 288         traces.append(plgo.Bar(x=[str(idx) + '.', ],
 289                                y=[y_val, ],
 290                                name=name,
 291                                text=hovertext,
 292                                hoverinfo="text"))
 293     try:
 294         # Create plot
 295         layout = deepcopy(plot["layout"])
 296         if layout.get("title", None):
 297             layout["title"] = "<b>Packet Throughput:</b> {0}". \
 298                 format(layout["title"])
 299         if y_max:
 300             layout["yaxis"]["range"] = [0, y_max + 1]
 301         plpl = plgo.Figure(data=traces, layout=layout)
 302         # Export Plot
 303         logging.info("    Writing file '{0}{1}'.".
 304                      format(plot["output-file"], plot["output-file-type"]))
 305         ploff.plot(plpl, show_link=False, auto_open=False,
 306                    filename='{0}{1}'.format(plot["output-file"],
 307                                             plot["output-file-type"]))
 308     except PlotlyError as err:
 309         logging.error("   Finished with error: {}".
 310                       format(repr(err).replace("\n", " ")))
 311         return
 312
 313
 314 def plot_soak_boxes(plot, input_data):
 315     """Generate the plot(s) with algorithm: plot_soak_boxes
 316     specified in the specification file.
 317
 318     :param plot: Plot to generate.
 319     :param input_data: Data to process.
 320     :type plot: pandas.Series
 321     :type input_data: InputData
 322     """
 323
 324     # Transform the data
 325     plot_title = plot.get("title", "")
 326     logging.info("    Creating the data set for the {0} '{1}'.".
 327                  format(plot.get("type", ""), plot_title))
 328     data = input_data.filter_data(plot)
 329     if data is None:
 330         logging.error("No data.")
 331         return
 332
 333     # Prepare the data for the plot
 334     y_vals = dict()
 335     y_tags = dict()
 336     for job in data:
 337         for build in job:
 338             for test in build:
 339                 if y_vals.get(test["parent"], None) is None:
 340                     y_tags[test["parent"]] = test.get("tags", None)
 341                 try:
 342                     if test["type"] in ("SOAK", ):
 343                         y_vals[test["parent"]] = test["throughput"]
 344                     else:
 345                         continue
 346                 except (KeyError, TypeError):
 347                     y_vals[test["parent"]] = dict()
 348
 349     # Sort the tests
 350     order = plot.get("sort", None)
 351     if order and y_tags:
 352         y_sorted = OrderedDict()
 353         y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()}
 354         for tag in order:
 355             logging.debug(tag)
 356             for suite, tags in y_tags_l.items():
 357                 if "not " in tag:
 358                     tag = tag.split(" ")[-1]
 359                     if tag.lower() in tags:
 360                         continue
 361                 else:
 362                     if tag.lower() not in tags:
 363                         continue
 364                 try:
 365                     y_sorted[suite] = y_vals.pop(suite)
 366                     y_tags_l.pop(suite)
 367                     logging.debug(suite)
 368                 except KeyError as err:
 369                     logging.error("Not found: {0}".format(repr(err)))
 370                 finally:
 371                     break
 372     else:
 373         y_sorted = y_vals
 374
 375     idx = 0
 376     y_max = 0
 377     traces = list()
 378     for test_name, test_data in y_sorted.items():
 379         idx += 1
 380         name = "{nr}. {name}".\
 381             format(nr=idx, name=test_name.lower().replace('-soak', '').
 382                    replace('2n1l-', ''))
 383         if len(name) > 55:
 384             name_lst = name.split('-')
 385             name = ""
 386             split_name = True
 387             for segment in name_lst:
 388                 if (len(name) + len(segment) + 1) > 55 and split_name:
 389                     name += "<br>    "
 390                     split_name = False
 391                 name += segment + '-'
 392             name = name[:-1]
 393
 394         y_val = test_data.get("UPPER", None)
 395         if y_val:
 396             y_val /= 1000000
 397             if y_val > y_max:
 398                 y_max = y_val
 399
 400         y_base = test_data.get("LOWER", None)
 401         if y_base:
 402             y_base /= 1000000
 403
 404         hovertext = ("Upper bound: {upper:.2f}<br>"
 405                      "Lower bound: {lower:.2f}".format(upper=y_val,
 406                                                            lower=y_base))
 407         traces.append(plgo.Bar(x=[str(idx) + '.', ],
 408                                # +0.05 to see the value in case lower == upper
 409                                y=[y_val - y_base + 0.05, ],
 410                                base=y_base,
 411                                name=name,
 412                                text=hovertext,
 413                                hoverinfo="text"))
 414     try:
 415         # Create plot
 416         layout = deepcopy(plot["layout"])
 417         if layout.get("title", None):
 418             layout["title"] = "<b>Throughput:</b> {0}". \
 419                 format(layout["title"])
 420         if y_max:
 421             layout["yaxis"]["range"] = [0, y_max + 1]
 422         plpl = plgo.Figure(data=traces, layout=layout)
 423         # Export Plot
 424         logging.info("    Writing file '{0}{1}'.".
 425                      format(plot["output-file"], plot["output-file-type"]))
 426         ploff.plot(plpl, show_link=False, auto_open=False,
 427                    filename='{0}{1}'.format(plot["output-file"],
 428                                             plot["output-file-type"]))
 429     except PlotlyError as err:
 430         logging.error("   Finished with error: {}".
 431                       format(repr(err).replace("\n", " ")))
 432         return
 433
 434
 435 def plot_latency_error_bars(plot, input_data):
 436     """Generate the plot(s) with algorithm: plot_latency_error_bars
 437     specified in the specification file.
 438
 439     :param plot: Plot to generate.
 440     :param input_data: Data to process.
 441     :type plot: pandas.Series
 442     :type input_data: InputData
 443     """
 444
 445     # Transform the data
 446     plot_title = plot.get("title", "")
 447     logging.info("    Creating the data set for the {0} '{1}'.".
 448                  format(plot.get("type", ""), plot_title))
 449     data = input_data.filter_data(plot)
 450     if data is None:
 451         logging.error("No data.")
 452         return
 453
 454     # Prepare the data for the plot
 455     y_tmp_vals = dict()
 456     y_tags = dict()
 457     for job in data:
 458         for build in job:
 459             for test in build:
 460                 try:
 461                     logging.debug("test['latency']: {0}\n".
 462                                  format(test["latency"]))
 463                 except ValueError as err:
 464                     logging.warning(repr(err))
 465                 if y_tmp_vals.get(test["parent"], None) is None:
 466                     y_tmp_vals[test["parent"]] = [
 467                         list(),  # direction1, min
 468                         list(),  # direction1, avg
 469                         list(),  # direction1, max
 470                         list(),  # direction2, min
 471                         list(),  # direction2, avg
 472                         list()   # direction2, max
 473                     ]
 474                     y_tags[test["parent"]] = test.get("tags", None)
 475                 try:
 476                     if test["type"] in ("NDRPDR", ):
 477                         if "-pdr" in plot_title.lower():
 478                             ttype = "PDR"
 479                         elif "-ndr" in plot_title.lower():
 480                             ttype = "NDR"
 481                         else:
 482                             logging.warning("Invalid test type: {0}".
 483                                             format(test["type"]))
 484                             continue
 485                         y_tmp_vals[test["parent"]][0].append(
 486                             test["latency"][ttype]["direction1"]["min"])
 487                         y_tmp_vals[test["parent"]][1].append(
 488                             test["latency"][ttype]["direction1"]["avg"])
 489                         y_tmp_vals[test["parent"]][2].append(
 490                             test["latency"][ttype]["direction1"]["max"])
 491                         y_tmp_vals[test["parent"]][3].append(
 492                             test["latency"][ttype]["direction2"]["min"])
 493                         y_tmp_vals[test["parent"]][4].append(
 494                             test["latency"][ttype]["direction2"]["avg"])
 495                         y_tmp_vals[test["parent"]][5].append(
 496                             test["latency"][ttype]["direction2"]["max"])
 497                     else:
 498                         logging.warning("Invalid test type: {0}".
 499                                         format(test["type"]))
 500                         continue
 501                 except (KeyError, TypeError) as err:
 502                     logging.warning(repr(err))
 503     logging.debug("y_tmp_vals: {0}\n".format(y_tmp_vals))
 504
 505     # Sort the tests
 506     order = plot.get("sort", None)
 507     if order and y_tags:
 508         y_sorted = OrderedDict()
 509         y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()}
 510         for tag in order:
 511             logging.debug(tag)
 512             for suite, tags in y_tags_l.items():
 513                 if "not " in tag:
 514                     tag = tag.split(" ")[-1]
 515                     if tag.lower() in tags:
 516                         continue
 517                 else:
 518                     if tag.lower() not in tags:
 519                         continue
 520                 try:
 521                     y_sorted[suite] = y_tmp_vals.pop(suite)
 522                     y_tags_l.pop(suite)
 523                     logging.debug(suite)
 524                 except KeyError as err:
 525                     logging.error("Not found: {0}".format(repr(err)))
 526                 finally:
 527                     break
 528     else:
 529         y_sorted = y_tmp_vals
 530
 531     logging.debug("y_sorted: {0}\n".format(y_sorted))
 532     x_vals = list()
 533     y_vals = list()
 534     y_mins = list()
 535     y_maxs = list()
 536     nr_of_samples = list()
 537     for key, val in y_sorted.items():
 538         name = re.sub(REGEX_NIC, "", key.replace('-ndrpdr', '').
 539                       replace('2n1l-', ''))
 540         x_vals.append(name)  # dir 1
 541         y_vals.append(mean(val[1]) if val[1] else None)
 542         y_mins.append(mean(val[0]) if val[0] else None)
 543         y_maxs.append(mean(val[2]) if val[2] else None)
 544         nr_of_samples.append(len(val[1]) if val[1] else 0)
 545         x_vals.append(name)  # dir 2
 546         y_vals.append(mean(val[4]) if val[4] else None)
 547         y_mins.append(mean(val[3]) if val[3] else None)
 548         y_maxs.append(mean(val[5]) if val[5] else None)
 549         nr_of_samples.append(len(val[3]) if val[3] else 0)
 550
 551     logging.debug("x_vals :{0}\n".format(x_vals))
 552     logging.debug("y_vals :{0}\n".format(y_vals))
 553     logging.debug("y_mins :{0}\n".format(y_mins))
 554     logging.debug("y_maxs :{0}\n".format(y_maxs))
 555     logging.debug("nr_of_samples :{0}\n".format(nr_of_samples))
 556     traces = list()
 557     annotations = list()
 558
 559     for idx in range(len(x_vals)):
 560         if not bool(int(idx % 2)):
 561             direction = "West-East"
 562         else:
 563             direction = "East-West"
 564         hovertext = ("No. of Runs: {nr}<br>"
 565                      "Test: {test}<br>"
 566                      "Direction: {dir}<br>".format(test=x_vals[idx],
 567                                                    dir=direction,
 568                                                    nr=nr_of_samples[idx]))
 569         if isinstance(y_maxs[idx], float):
 570             hovertext += "Max: {max:.2f}uSec<br>".format(max=y_maxs[idx])
 571         if isinstance(y_vals[idx], float):
 572             hovertext += "Mean: {avg:.2f}uSec<br>".format(avg=y_vals[idx])
 573         if isinstance(y_mins[idx], float):
 574             hovertext += "Min: {min:.2f}uSec".format(min=y_mins[idx])
 575
 576         if isinstance(y_maxs[idx], float) and isinstance(y_vals[idx], float):
 577             array = [y_maxs[idx] - y_vals[idx], ]
 578         else:
 579             array = [None, ]
 580         if isinstance(y_mins[idx], float) and isinstance(y_vals[idx], float):
 581             arrayminus = [y_vals[idx] - y_mins[idx], ]
 582         else:
 583             arrayminus = [None, ]
 584         logging.debug("y_vals[{1}] :{0}\n".format(y_vals[idx], idx))
 585         logging.debug("array :{0}\n".format(array))
 586         logging.debug("arrayminus :{0}\n".format(arrayminus))
 587         traces.append(plgo.Scatter(
 588             x=[idx, ],
 589             y=[y_vals[idx], ],
 590             name=x_vals[idx],
 591             legendgroup=x_vals[idx],
 592             showlegend=bool(int(idx % 2)),
 593             mode="markers",
 594             error_y=dict(
 595                 type='data',
 596                 symmetric=False,
 597                 array=array,
 598                 arrayminus=arrayminus,
 599                 color=COLORS[int(idx / 2)]
 600             ),
 601             marker=dict(
 602                 size=10,
 603                 color=COLORS[int(idx / 2)],
 604             ),
 605             text=hovertext,
 606             hoverinfo="text",
 607         ))
 608         annotations.append(dict(
 609             x=idx,
 610             y=0,
 611             xref="x",
 612             yref="y",
 613             xanchor="center",
 614             yanchor="top",
 615             text="E-W" if bool(int(idx % 2)) else "W-E",
 616             font=dict(
 617                 size=16,
 618             ),
 619             align="center",
 620             showarrow=False
 621         ))
 622
 623     try:
 624         # Create plot
 625         logging.info("    Writing file '{0}{1}'.".
 626                      format(plot["output-file"], plot["output-file-type"]))
 627         layout = deepcopy(plot["layout"])
 628         if layout.get("title", None):
 629             layout["title"] = "<b>Latency:</b> {0}".\
 630                 format(layout["title"])
 631         layout["annotations"] = annotations
 632         plpl = plgo.Figure(data=traces, layout=layout)
 633
 634         # Export Plot
 635         ploff.plot(plpl,
 636                    show_link=False, auto_open=False,
 637                    filename='{0}{1}'.format(plot["output-file"],
 638                                             plot["output-file-type"]))
 639     except PlotlyError as err:
 640         logging.error("   Finished with error: {}".
 641                       format(str(err).replace("\n", " ")))
 642         return
 643
 644
 645 def plot_throughput_speedup_analysis(plot, input_data):
 646     """Generate the plot(s) with algorithm:
 647     plot_throughput_speedup_analysis
 648     specified in the specification file.
 649
 650     :param plot: Plot to generate.
 651     :param input_data: Data to process.
 652     :type plot: pandas.Series
 653     :type input_data: InputData
 654     """
 655
 656     # Transform the data
 657     plot_title = plot.get("title", "")
 658     logging.info("    Creating the data set for the {0} '{1}'.".
 659                  format(plot.get("type", ""), plot_title))
 660     data = input_data.filter_data(plot)
 661     if data is None:
 662         logging.error("No data.")
 663         return
 664
 665     y_vals = dict()
 666     y_tags = dict()
 667     for job in data:
 668         for build in job:
 669             for test in build:
 670                 if y_vals.get(test["parent"], None) is None:
 671                     y_vals[test["parent"]] = {"1": list(),
 672                                               "2": list(),
 673                                               "4": list()}
 674                     y_tags[test["parent"]] = test.get("tags", None)
 675                 try:
 676                     if test["type"] in ("NDRPDR",):
 677                         if "-pdr" in plot_title.lower():
 678                             ttype = "PDR"
 679                         elif "-ndr" in plot_title.lower():
 680                             ttype = "NDR"
 681                         else:
 682                             continue
 683                         if "1C" in test["tags"]:
 684                             y_vals[test["parent"]]["1"]. \
 685                                 append(test["throughput"][ttype]["LOWER"])
 686                         elif "2C" in test["tags"]:
 687                             y_vals[test["parent"]]["2"]. \
 688                                 append(test["throughput"][ttype]["LOWER"])
 689                         elif "4C" in test["tags"]:
 690                             y_vals[test["parent"]]["4"]. \
 691                                 append(test["throughput"][ttype]["LOWER"])
 692                 except (KeyError, TypeError):
 693                     pass
 694
 695     if not y_vals:
 696         logging.warning("No data for the plot '{}'".
 697                         format(plot.get("title", "")))
 698         return
 699
 700     y_1c_max = dict()
 701     for test_name, test_vals in y_vals.items():
 702         for key, test_val in test_vals.items():
 703             if test_val:
 704                 avg_val = sum(test_val) / len(test_val)
 705                 y_vals[test_name][key] = (avg_val, len(test_val))
 706                 ideal = avg_val / (int(key) * 1000000.0)
 707                 if test_name not in y_1c_max or ideal > y_1c_max[test_name]:
 708                     y_1c_max[test_name] = ideal
 709
 710     vals = dict()
 711     y_max = list()
 712     nic_limit = 0
 713     lnk_limit = 0
 714     pci_limit = plot["limits"]["pci"]["pci-g3-x8"]
 715     for test_name, test_vals in y_vals.items():
 716         try:
 717             if test_vals["1"][1]:
 718                 name = re.sub(REGEX_NIC, "", test_name.replace('-ndrpdr', '').
 719                               replace('2n1l-', ''))
 720                 vals[name] = dict()
 721                 y_val_1 = test_vals["1"][0] / 1000000.0
 722                 y_val_2 = test_vals["2"][0] / 1000000.0 if test_vals["2"][0] \
 723                     else None
 724                 y_val_4 = test_vals["4"][0] / 1000000.0 if test_vals["4"][0] \
 725                     else None
 726
 727                 vals[name]["val"] = [y_val_1, y_val_2, y_val_4]
 728                 vals[name]["rel"] = [1.0, None, None]
 729                 vals[name]["ideal"] = [y_1c_max[test_name],
 730                                        y_1c_max[test_name] * 2,
 731                                        y_1c_max[test_name] * 4]
 732                 vals[name]["diff"] = [(y_val_1 - y_1c_max[test_name]) * 100 /
 733                                       y_val_1, None, None]
 734                 vals[name]["count"] = [test_vals["1"][1],
 735                                        test_vals["2"][1],
 736                                        test_vals["4"][1]]
 737
 738                 try:
 739                     # val_max = max(max(vals[name]["val"], vals[name]["ideal"]))
 740                     val_max = max(vals[name]["val"])
 741                 except ValueError as err:
 742                     logging.error(err)
 743                     continue
 744                 if val_max:
 745                     # y_max.append(int((val_max / 10) + 1) * 10)
 746                     y_max.append(val_max)
 747
 748                 if y_val_2:
 749                     vals[name]["rel"][1] = round(y_val_2 / y_val_1, 2)
 750                     vals[name]["diff"][1] = \
 751                         (y_val_2 - vals[name]["ideal"][1]) * 100 / y_val_2
 752                 if y_val_4:
 753                     vals[name]["rel"][2] = round(y_val_4 / y_val_1, 2)
 754                     vals[name]["diff"][2] = \
 755                         (y_val_4 - vals[name]["ideal"][2]) * 100 / y_val_4
 756         except IndexError as err:
 757             logging.warning("No data for '{0}'".format(test_name))
 758             logging.warning(repr(err))
 759
 760         # Limits:
 761         if "x520" in test_name:
 762             limit = plot["limits"]["nic"]["x520"]
 763         elif "x710" in test_name:
 764             limit = plot["limits"]["nic"]["x710"]
 765         elif "xxv710" in test_name:
 766             limit = plot["limits"]["nic"]["xxv710"]
 767         elif "xl710" in test_name:
 768             limit = plot["limits"]["nic"]["xl710"]
 769         elif "x553" in test_name:
 770             limit = plot["limits"]["nic"]["x553"]
 771         else:
 772             limit = 0
 773         if limit > nic_limit:
 774             nic_limit = limit
 775
 776         mul = 2 if "ge2p" in test_name else 1
 777         if "10ge" in test_name:
 778             limit = plot["limits"]["link"]["10ge"] * mul
 779         elif "25ge" in test_name:
 780             limit = plot["limits"]["link"]["25ge"] * mul
 781         elif "40ge" in test_name:
 782             limit = plot["limits"]["link"]["40ge"] * mul
 783         elif "100ge" in test_name:
 784             limit = plot["limits"]["link"]["100ge"] * mul
 785         else:
 786             limit = 0
 787         if limit > lnk_limit:
 788             lnk_limit = limit
 789
 790     # Sort the tests
 791     order = plot.get("sort", None)
 792     if order and y_tags:
 793         y_sorted = OrderedDict()
 794         y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()}
 795         for tag in order:
 796             for test, tags in y_tags_l.items():
 797                 if tag.lower() in tags:
 798                     name = re.sub(REGEX_NIC, "",
 799                                   test.replace('-ndrpdr', '').
 800                                   replace('2n1l-', ''))
 801                     try:
 802                         y_sorted[name] = vals.pop(name)
 803                         y_tags_l.pop(test)
 804                     except KeyError as err:
 805                         logging.error("Not found: {0}".format(err))
 806                     finally:
 807                         break
 808     else:
 809         y_sorted = vals
 810
 811     traces = list()
 812     annotations = list()
 813     x_vals = [1, 2, 4]
 814
 815     # Limits:
 816     try:
 817         threshold = 1.1 * max(y_max)  # 10%
 818     except ValueError as err:
 819         logging.error(err)
 820         return
 821     nic_limit /= 1000000.0
 822     # if nic_limit < threshold:
 823     traces.append(plgo.Scatter(
 824         x=x_vals,
 825         y=[nic_limit, ] * len(x_vals),
 826         name="NIC: {0:.2f}Mpps".format(nic_limit),
 827         showlegend=False,
 828         mode="lines",
 829         line=dict(
 830             dash="dot",
 831             color=COLORS[-1],
 832             width=1),
 833         hoverinfo="none"
 834     ))
 835     annotations.append(dict(
 836         x=1,
 837         y=nic_limit,
 838         xref="x",
 839         yref="y",
 840         xanchor="left",
 841         yanchor="bottom",
 842         text="NIC: {0:.2f}Mpps".format(nic_limit),
 843         font=dict(
 844             size=14,
 845             color=COLORS[-1],
 846         ),
 847         align="left",
 848         showarrow=False
 849     ))
 850     # y_max.append(int((nic_limit / 10) + 1) * 10)
 851     y_max.append(nic_limit)
 852
 853     lnk_limit /= 1000000.0
 854     if lnk_limit < threshold:
 855         traces.append(plgo.Scatter(
 856             x=x_vals,
 857             y=[lnk_limit, ] * len(x_vals),
 858             name="Link: {0:.2f}Mpps".format(lnk_limit),
 859             showlegend=False,
 860             mode="lines",
 861             line=dict(
 862                 dash="dot",
 863                 color=COLORS[-2],
 864                 width=1),
 865             hoverinfo="none"
 866         ))
 867         annotations.append(dict(
 868             x=1,
 869             y=lnk_limit,
 870             xref="x",
 871             yref="y",
 872             xanchor="left",
 873             yanchor="bottom",
 874             text="Link: {0:.2f}Mpps".format(lnk_limit),
 875             font=dict(
 876                 size=14,
 877                 color=COLORS[-2],
 878             ),
 879             align="left",
 880             showarrow=False
 881         ))
 882         # y_max.append(int((lnk_limit / 10) + 1) * 10)
 883         y_max.append(lnk_limit)
 884
 885     pci_limit /= 1000000.0
 886     if (pci_limit < threshold and
 887         (pci_limit < lnk_limit * 0.95 or lnk_limit > lnk_limit * 1.05)):
 888         traces.append(plgo.Scatter(
 889             x=x_vals,
 890             y=[pci_limit, ] * len(x_vals),
 891             name="PCIe: {0:.2f}Mpps".format(pci_limit),
 892             showlegend=False,
 893             mode="lines",
 894             line=dict(
 895                 dash="dot",
 896                 color=COLORS[-3],
 897                 width=1),
 898             hoverinfo="none"
 899         ))
 900         annotations.append(dict(
 901             x=1,
 902             y=pci_limit,
 903             xref="x",
 904             yref="y",
 905             xanchor="left",
 906             yanchor="bottom",
 907             text="PCIe: {0:.2f}Mpps".format(pci_limit),
 908             font=dict(
 909                 size=14,
 910                 color=COLORS[-3],
 911             ),
 912             align="left",
 913             showarrow=False
 914         ))
 915         # y_max.append(int((pci_limit / 10) + 1) * 10)
 916         y_max.append(pci_limit)
 917
 918     # Perfect and measured:
 919     cidx = 0
 920     for name, val in y_sorted.iteritems():
 921         hovertext = list()
 922         try:
 923             for idx in range(len(val["val"])):
 924                 htext = ""
 925                 if isinstance(val["val"][idx], float):
 926                     htext += "No. of Runs: {1}<br>" \
 927                              "Mean: {0:.2f}Mpps<br>".format(val["val"][idx],
 928                                                             val["count"][idx])
 929                 if isinstance(val["diff"][idx], float):
 930                     htext += "Diff: {0:.0f}%<br>".format(round(val["diff"][idx]))
 931                 if isinstance(val["rel"][idx], float):
 932                     htext += "Speedup: {0:.2f}".format(val["rel"][idx])
 933                 hovertext.append(htext)
 934             traces.append(plgo.Scatter(x=x_vals,
 935                                        y=val["val"],
 936                                        name=name,
 937                                        legendgroup=name,
 938                                        mode="lines+markers",
 939                                        line=dict(
 940                                            color=COLORS[cidx],
 941                                            width=2),
 942                                        marker=dict(
 943                                            symbol="circle",
 944                                            size=10
 945                                        ),
 946                                        text=hovertext,
 947                                        hoverinfo="text+name"
 948                                        ))
 949             traces.append(plgo.Scatter(x=x_vals,
 950                                        y=val["ideal"],
 951                                        name="{0} perfect".format(name),
 952                                        legendgroup=name,
 953                                        showlegend=False,
 954                                        mode="lines",
 955                                        line=dict(
 956                                            color=COLORS[cidx],
 957                                            width=2,
 958                                            dash="dash"),
 959                                        text=["Perfect: {0:.2f}Mpps".format(y)
 960                                              for y in val["ideal"]],
 961                                        hoverinfo="text"
 962                                        ))
 963             cidx += 1
 964         except (IndexError, ValueError, KeyError) as err:
 965             logging.warning("No data for '{0}'".format(name))
 966             logging.warning(repr(err))
 967
 968     try:
 969         # Create plot
 970         logging.info("    Writing file '{0}{1}'.".
 971                      format(plot["output-file"], plot["output-file-type"]))
 972         layout = deepcopy(plot["layout"])
 973         if layout.get("title", None):
 974             layout["title"] = "<b>Speedup Multi-core:</b> {0}". \
 975                 format(layout["title"])
 976         # layout["yaxis"]["range"] = [0, int((max(y_max) / 10) + 1) * 10]
 977         layout["yaxis"]["range"] = [0, int(max(y_max) * 1.1)]
 978         layout["annotations"].extend(annotations)
 979         plpl = plgo.Figure(data=traces, layout=layout)
 980
 981         # Export Plot
 982         ploff.plot(plpl,
 983                    show_link=False, auto_open=False,
 984                    filename='{0}{1}'.format(plot["output-file"],
 985                                             plot["output-file-type"]))
 986     except PlotlyError as err:
 987         logging.error("   Finished with error: {}".
 988                       format(str(err).replace("\n", " ")))
 989         return
 990
 991
 992 def plot_http_server_performance_box(plot, input_data):
 993     """Generate the plot(s) with algorithm: plot_http_server_performance_box
 994     specified in the specification file.
 995
 996     :param plot: Plot to generate.
 997     :param input_data: Data to process.
 998     :type plot: pandas.Series
 999     :type input_data: InputData
1000     """
1001
1002     # Transform the data
1003     logging.info("    Creating the data set for the {0} '{1}'.".
1004                  format(plot.get("type", ""), plot.get("title", "")))
1005     data = input_data.filter_data(plot)
1006     if data is None:
1007         logging.error("No data.")
1008         return
1009
1010     # Prepare the data for the plot
1011     y_vals = dict()
1012     for job in data:
1013         for build in job:
1014             for test in build:
1015                 if y_vals.get(test["name"], None) is None:
1016                     y_vals[test["name"]] = list()
1017                 try:
1018                     y_vals[test["name"]].append(test["result"])
1019                 except (KeyError, TypeError):
1020                     y_vals[test["name"]].append(None)
1021
1022     # Add None to the lists with missing data
1023     max_len = 0
1024     nr_of_samples = list()
1025     for val in y_vals.values():
1026         if len(val) > max_len:
1027             max_len = len(val)
1028         nr_of_samples.append(len(val))
1029     for key, val in y_vals.items():
1030         if len(val) < max_len:
1031             val.extend([None for _ in range(max_len - len(val))])
1032
1033     # Add plot traces
1034     traces = list()
1035     df = pd.DataFrame(y_vals)
1036     df.head()
1037     for i, col in enumerate(df.columns):
1038         name = "{nr}. ({samples:02d} run{plural}) {name}".\
1039             format(nr=(i + 1),
1040                    samples=nr_of_samples[i],
1041                    plural='s' if nr_of_samples[i] > 1 else '',
1042                    name=col.lower().replace('-ndrpdr', ''))
1043         if len(name) > 50:
1044             name_lst = name.split('-')
1045             name = ""
1046             split_name = True
1047             for segment in name_lst:
1048                 if (len(name) + len(segment) + 1) > 50 and split_name:
1049                     name += "<br>    "
1050                     split_name = False
1051                 name += segment + '-'
1052             name = name[:-1]
1053
1054         traces.append(plgo.Box(x=[str(i + 1) + '.'] * len(df[col]),
1055                                y=df[col],
1056                                name=name,
1057                                **plot["traces"]))
1058     try:
1059         # Create plot
1060         plpl = plgo.Figure(data=traces, layout=plot["layout"])
1061
1062         # Export Plot
1063         logging.info("    Writing file '{0}{1}'.".
1064                      format(plot["output-file"], plot["output-file-type"]))
1065         ploff.plot(plpl, show_link=False, auto_open=False,
1066                    filename='{0}{1}'.format(plot["output-file"],
1067                                             plot["output-file-type"]))
1068     except PlotlyError as err:
1069         logging.error("   Finished with error: {}".
1070                       format(str(err).replace("\n", " ")))
1071         return
1072
1073
1074 def plot_service_density_heatmap(plot, input_data):
1075     """Generate the plot(s) with algorithm: plot_service_density_heatmap
1076     specified in the specification file.
1077
1078     :param plot: Plot to generate.
1079     :param input_data: Data to process.
1080     :type plot: pandas.Series
1081     :type input_data: InputData
1082     """
1083
1084     REGEX_CN = re.compile(r'^(\d*)R(\d*)C$')
1085     REGEX_TEST_NAME = re.compile(r'^.*-(\d+vhost|\d+memif)-'
1086                                  r'(\d+chain|\d+pipe)-'
1087                                  r'(\d+vm|\d+dcr|\d+drc).*$')
1088
1089     txt_chains = list()
1090     txt_nodes = list()
1091     vals = dict()
1092
1093     # Transform the data
1094     logging.info("    Creating the data set for the {0} '{1}'.".
1095                  format(plot.get("type", ""), plot.get("title", "")))
1096     data = input_data.filter_data(plot, continue_on_error=True)
1097     if data is None or data.empty:
1098         logging.error("No data.")
1099         return
1100
1101     for job in data:
1102         for build in job:
1103             for test in build:
1104                 for tag in test['tags']:
1105                     groups = re.search(REGEX_CN, tag)
1106                     if groups:
1107                         c = str(groups.group(1))
1108                         n = str(groups.group(2))
1109                         break
1110                 else:
1111                     continue
1112                 groups = re.search(REGEX_TEST_NAME, test["name"])
1113                 if groups and len(groups.groups()) == 3:
1114                     hover_name = "{vhost}-{chain}-{vm}".format(
1115                         vhost=str(groups.group(1)),
1116                         chain=str(groups.group(2)),
1117                         vm=str(groups.group(3)))
1118                 else:
1119                     hover_name = ""
1120                 if vals.get(c, None) is None:
1121                     vals[c] = dict()
1122                 if vals[c].get(n, None) is None:
1123                     vals[c][n] = dict(name=hover_name,
1124                                       vals=list(),
1125                                       nr=None,
1126                                       mean=None,
1127                                       stdev=None)
1128                 if plot["include-tests"] == "MRR":
1129                     result = test["result"]["receive-rate"].avg
1130                 elif plot["include-tests"] == "PDR":
1131                     result = test["throughput"]["PDR"]["LOWER"]
1132                 elif plot["include-tests"] == "NDR":
1133                     result = test["throughput"]["NDR"]["LOWER"]
1134                 else:
1135                     result = None
1136
1137                 if result:
1138                     vals[c][n]["vals"].append(result)
1139
1140     if not vals:
1141         logging.error("No data.")
1142         return
1143
1144     for key_c in vals.keys():
1145         txt_chains.append(key_c)
1146         for key_n in vals[key_c].keys():
1147             txt_nodes.append(key_n)
1148             if vals[key_c][key_n]["vals"]:
1149                 vals[key_c][key_n]["nr"] = len(vals[key_c][key_n]["vals"])
1150                 vals[key_c][key_n]["mean"] = \
1151                     round(mean(vals[key_c][key_n]["vals"]) / 1000000, 1)
1152                 vals[key_c][key_n]["stdev"] = \
1153                     round(stdev(vals[key_c][key_n]["vals"]) / 1000000, 1)
1154     txt_nodes = list(set(txt_nodes))
1155
1156     txt_chains = sorted(txt_chains, key=lambda chain: int(chain))
1157     txt_nodes = sorted(txt_nodes, key=lambda node: int(node))
1158
1159     chains = [i + 1 for i in range(len(txt_chains))]
1160     nodes = [i + 1 for i in range(len(txt_nodes))]
1161
1162     data = [list() for _ in range(len(chains))]
1163     for c in chains:
1164         for n in nodes:
1165             try:
1166                 val = vals[txt_chains[c - 1]][txt_nodes[n - 1]]["mean"]
1167             except (KeyError, IndexError):
1168                 val = None
1169             data[c - 1].append(val)
1170
1171     # Colorscales:
1172     my_green = [[0.0, 'rgb(235, 249, 242)'],
1173                 [1.0, 'rgb(45, 134, 89)']]
1174
1175     my_blue = [[0.0, 'rgb(236, 242, 248)'],
1176                [1.0, 'rgb(57, 115, 172)']]
1177
1178     my_grey = [[0.0, 'rgb(230, 230, 230)'],
1179                [1.0, 'rgb(102, 102, 102)']]
1180
1181     hovertext = list()
1182     annotations = list()
1183
1184     text = ("Test: {name}<br>"
1185             "Runs: {nr}<br>"
1186             "Thput: {val}<br>"
1187             "StDev: {stdev}")
1188
1189     for c in range(len(txt_chains)):
1190         hover_line = list()
1191         for n in range(len(txt_nodes)):
1192             if data[c][n] is not None:
1193                 annotations.append(dict(
1194                     x=n+1,
1195                     y=c+1,
1196                     xref="x",
1197                     yref="y",
1198                     xanchor="center",
1199                     yanchor="middle",
1200                     text=str(data[c][n]),
1201                     font=dict(
1202                         size=14,
1203                     ),
1204                     align="center",
1205                     showarrow=False
1206                 ))
1207                 hover_line.append(text.format(
1208                     name=vals[txt_chains[c]][txt_nodes[n]]["name"],
1209                     nr=vals[txt_chains[c]][txt_nodes[n]]["nr"],
1210                     val=data[c][n],
1211                     stdev=vals[txt_chains[c]][txt_nodes[n]]["stdev"]))
1212         hovertext.append(hover_line)
1213
1214     traces = [
1215         plgo.Heatmap(x=nodes,
1216                      y=chains,
1217                      z=data,
1218                      colorbar=dict(
1219                          title=plot.get("z-axis", ""),
1220                          titleside="right",
1221                          titlefont=dict(
1222                             size=16
1223                          ),
1224                          tickfont=dict(
1225                              size=16,
1226                          ),
1227                          tickformat=".1f",
1228                          yanchor="bottom",
1229                          y=-0.02,
1230                          len=0.925,
1231                      ),
1232                      showscale=True,
1233                      colorscale=my_green,
1234                      text=hovertext,
1235                      hoverinfo="text")
1236     ]
1237
1238     for idx, item in enumerate(txt_nodes):
1239         # X-axis, numbers:
1240         annotations.append(dict(
1241             x=idx+1,
1242             y=0.05,
1243             xref="x",
1244             yref="y",
1245             xanchor="center",
1246             yanchor="top",
1247             text=item,
1248             font=dict(
1249                 size=16,
1250             ),
1251             align="center",
1252             showarrow=False
1253         ))
1254     for idx, item in enumerate(txt_chains):
1255         # Y-axis, numbers:
1256         annotations.append(dict(
1257             x=0.35,
1258             y=idx+1,
1259             xref="x",
1260             yref="y",
1261             xanchor="right",
1262             yanchor="middle",
1263             text=item,
1264             font=dict(
1265                 size=16,
1266             ),
1267             align="center",
1268             showarrow=False
1269         ))
1270     # X-axis, title:
1271     annotations.append(dict(
1272         x=0.55,
1273         y=-0.15,
1274         xref="paper",
1275         yref="y",
1276         xanchor="center",
1277         yanchor="bottom",
1278         text=plot.get("x-axis", ""),
1279         font=dict(
1280             size=16,
1281         ),
1282         align="center",
1283         showarrow=False
1284     ))
1285     # Y-axis, title:
1286     annotations.append(dict(
1287         x=-0.1,
1288         y=0.5,
1289         xref="x",
1290         yref="paper",
1291         xanchor="center",
1292         yanchor="middle",
1293         text=plot.get("y-axis", ""),
1294         font=dict(
1295             size=16,
1296         ),
1297         align="center",
1298         textangle=270,
1299         showarrow=False
1300     ))
1301     updatemenus = list([
1302         dict(
1303             x=1.0,
1304             y=0.0,
1305             xanchor='right',
1306             yanchor='bottom',
1307             direction='up',
1308             buttons=list([
1309                 dict(
1310                     args=[{"colorscale": [my_green, ], "reversescale": False}],
1311                     label="Green",
1312                     method="update"
1313                 ),
1314                 dict(
1315                     args=[{"colorscale": [my_blue, ], "reversescale": False}],
1316                     label="Blue",
1317                     method="update"
1318                 ),
1319                 dict(
1320                     args=[{"colorscale": [my_grey, ], "reversescale": False}],
1321                     label="Grey",
1322                     method="update"
1323                 )
1324             ])
1325         )
1326     ])
1327
1328     try:
1329         layout = deepcopy(plot["layout"])
1330     except KeyError as err:
1331         logging.error("Finished with error: No layout defined")
1332         logging.error(repr(err))
1333         return
1334
1335     layout["annotations"] = annotations
1336     layout['updatemenus'] = updatemenus
1337
1338     try:
1339         # Create plot
1340         plpl = plgo.Figure(data=traces, layout=layout)
1341
1342         # Export Plot
1343         logging.info("    Writing file '{0}{1}'.".
1344                      format(plot["output-file"], plot["output-file-type"]))
1345         ploff.plot(plpl, show_link=False, auto_open=False,
1346                    filename='{0}{1}'.format(plot["output-file"],
1347                                             plot["output-file-type"]))
1348     except PlotlyError as err:
1349         logging.error("   Finished with error: {}".
1350                       format(str(err).replace("\n", " ")))
1351         return