resources/tools/presentation/new/generator_tables.py

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import prettytable
  21 import pandas as pd
  22
  23 from string import replace
  24 from collections import OrderedDict
  25 from numpy import nan, isnan
  26 from xml.etree import ElementTree as ET
  27
  28 from errors import PresentationError
  29 from utils import mean, stdev, relative_change, classify_anomalies
  30
  31
  32 def generate_tables(spec, data):
  33     """Generate all tables specified in the specification file.
  34
  35     :param spec: Specification read from the specification file.
  36     :param data: Data to process.
  37     :type spec: Specification
  38     :type data: InputData
  39     """
  40
  41     logging.info("Generating the tables ...")
  42     for table in spec.tables:
  43         try:
  44             eval(table["algorithm"])(table, data)
  45         except NameError as err:
  46             logging.error("Probably algorithm '{alg}' is not defined: {err}".
  47                           format(alg=table["algorithm"], err=repr(err)))
  48     logging.info("Done.")
  49
  50
  51 def table_details(table, input_data):
  52     """Generate the table(s) with algorithm: table_detailed_test_results
  53     specified in the specification file.
  54
  55     :param table: Table to generate.
  56     :param input_data: Data to process.
  57     :type table: pandas.Series
  58     :type input_data: InputData
  59     """
  60
  61     logging.info("  Generating the table {0} ...".
  62                  format(table.get("title", "")))
  63
  64     # Transform the data
  65     logging.info("    Creating the data set for the {0} '{1}'.".
  66                  format(table.get("type", ""), table.get("title", "")))
  67     data = input_data.filter_data(table)
  68
  69     # Prepare the header of the tables
  70     header = list()
  71     for column in table["columns"]:
  72         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  73
  74     # Generate the data for the table according to the model in the table
  75     # specification
  76     job = table["data"].keys()[0]
  77     build = str(table["data"][job][0])
  78     try:
  79         suites = input_data.suites(job, build)
  80     except KeyError:
  81         logging.error("    No data available. The table will not be generated.")
  82         return
  83
  84     for suite_longname, suite in suites.iteritems():
  85         # Generate data
  86         suite_name = suite["name"]
  87         table_lst = list()
  88         for test in data[job][build].keys():
  89             if data[job][build][test]["parent"] in suite_name:
  90                 row_lst = list()
  91                 for column in table["columns"]:
  92                     try:
  93                         col_data = str(data[job][build][test][column["data"].
  94                                        split(" ")[1]]).replace('"', '""')
  95                         if column["data"].split(" ")[1] in ("vat-history",
  96                                                             "show-run"):
  97                             col_data = replace(col_data, " |br| ", "",
  98                                                maxreplace=1)
  99                             col_data = " |prein| {0} |preout| ".\
 100                                 format(col_data[:-5])
 101                         row_lst.append('"{0}"'.format(col_data))
 102                     except KeyError:
 103                         row_lst.append("No data")
 104                 table_lst.append(row_lst)
 105
 106         # Write the data to file
 107         if table_lst:
 108             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 109                                             table["output-file-ext"])
 110             logging.info("      Writing file: '{}'".format(file_name))
 111             with open(file_name, "w") as file_handler:
 112                 file_handler.write(",".join(header) + "\n")
 113                 for item in table_lst:
 114                     file_handler.write(",".join(item) + "\n")
 115
 116     logging.info("  Done.")
 117
 118
 119 def table_merged_details(table, input_data):
 120     """Generate the table(s) with algorithm: table_merged_details
 121     specified in the specification file.
 122
 123     :param table: Table to generate.
 124     :param input_data: Data to process.
 125     :type table: pandas.Series
 126     :type input_data: InputData
 127     """
 128
 129     logging.info("  Generating the table {0} ...".
 130                  format(table.get("title", "")))
 131
 132     # Transform the data
 133     logging.info("    Creating the data set for the {0} '{1}'.".
 134                  format(table.get("type", ""), table.get("title", "")))
 135     data = input_data.filter_data(table)
 136     data = input_data.merge_data(data)
 137     data.sort_index(inplace=True)
 138
 139     logging.info("    Creating the data set for the {0} '{1}'.".
 140                  format(table.get("type", ""), table.get("title", "")))
 141     suites = input_data.filter_data(table, data_set="suites")
 142     suites = input_data.merge_data(suites)
 143
 144     # Prepare the header of the tables
 145     header = list()
 146     for column in table["columns"]:
 147         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 148
 149     for _, suite in suites.iteritems():
 150         # Generate data
 151         suite_name = suite["name"]
 152         table_lst = list()
 153         for test in data.keys():
 154             if data[test]["parent"] in suite_name:
 155                 row_lst = list()
 156                 for column in table["columns"]:
 157                     try:
 158                         col_data = str(data[test][column["data"].
 159                                        split(" ")[1]]).replace('"', '""')
 160                         if column["data"].split(" ")[1] in ("vat-history",
 161                                                             "show-run"):
 162                             col_data = replace(col_data, " |br| ", "",
 163                                                maxreplace=1)
 164                             col_data = " |prein| {0} |preout| ".\
 165                                 format(col_data[:-5])
 166                         row_lst.append('"{0}"'.format(col_data))
 167                     except KeyError:
 168                         row_lst.append("No data")
 169                 table_lst.append(row_lst)
 170
 171         # Write the data to file
 172         if table_lst:
 173             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 174                                             table["output-file-ext"])
 175             logging.info("      Writing file: '{}'".format(file_name))
 176             with open(file_name, "w") as file_handler:
 177                 file_handler.write(",".join(header) + "\n")
 178                 for item in table_lst:
 179                     file_handler.write(",".join(item) + "\n")
 180
 181     logging.info("  Done.")
 182
 183
 184 def table_performance_improvements(table, input_data):
 185     """Generate the table(s) with algorithm: table_performance_improvements
 186     specified in the specification file.
 187
 188     :param table: Table to generate.
 189     :param input_data: Data to process.
 190     :type table: pandas.Series
 191     :type input_data: InputData
 192     """
 193
 194     def _write_line_to_file(file_handler, data):
 195         """Write a line to the .csv file.
 196
 197         :param file_handler: File handler for the csv file. It must be open for
 198          writing text.
 199         :param data: Item to be written to the file.
 200         :type file_handler: BinaryIO
 201         :type data: list
 202         """
 203
 204         line_lst = list()
 205         for item in data:
 206             if isinstance(item["data"], str):
 207                 # Remove -?drdisc from the end
 208                 if item["data"].endswith("drdisc"):
 209                     item["data"] = item["data"][:-8]
 210                 line_lst.append(item["data"])
 211             elif isinstance(item["data"], float):
 212                 line_lst.append("{:.1f}".format(item["data"]))
 213             elif item["data"] is None:
 214                 line_lst.append("")
 215         file_handler.write(",".join(line_lst) + "\n")
 216
 217     logging.info("  Generating the table {0} ...".
 218                  format(table.get("title", "")))
 219
 220     # Read the template
 221     file_name = table.get("template", None)
 222     if file_name:
 223         try:
 224             tmpl = _read_csv_template(file_name)
 225         except PresentationError:
 226             logging.error("  The template '{0}' does not exist. Skipping the "
 227                           "table.".format(file_name))
 228             return None
 229     else:
 230         logging.error("The template is not defined. Skipping the table.")
 231         return None
 232
 233     # Transform the data
 234     logging.info("    Creating the data set for the {0} '{1}'.".
 235                  format(table.get("type", ""), table.get("title", "")))
 236     data = input_data.filter_data(table)
 237
 238     # Prepare the header of the tables
 239     header = list()
 240     for column in table["columns"]:
 241         header.append(column["title"])
 242
 243     # Generate the data for the table according to the model in the table
 244     # specification
 245     tbl_lst = list()
 246     for tmpl_item in tmpl:
 247         tbl_item = list()
 248         for column in table["columns"]:
 249             cmd = column["data"].split(" ")[0]
 250             args = column["data"].split(" ")[1:]
 251             if cmd == "template":
 252                 try:
 253                     val = float(tmpl_item[int(args[0])])
 254                 except ValueError:
 255                     val = tmpl_item[int(args[0])]
 256                 tbl_item.append({"data": val})
 257             elif cmd == "data":
 258                 jobs = args[0:-1]
 259                 operation = args[-1]
 260                 data_lst = list()
 261                 for job in jobs:
 262                     for build in data[job]:
 263                         try:
 264                             data_lst.append(float(build[tmpl_item[0]]
 265                                                   ["throughput"]["value"]))
 266                         except (KeyError, TypeError):
 267                             # No data, ignore
 268                             continue
 269                 if data_lst:
 270                     tbl_item.append({"data": (eval(operation)(data_lst)) /
 271                                              1000000})
 272                 else:
 273                     tbl_item.append({"data": None})
 274             elif cmd == "operation":
 275                 operation = args[0]
 276                 try:
 277                     nr1 = float(tbl_item[int(args[1])]["data"])
 278                     nr2 = float(tbl_item[int(args[2])]["data"])
 279                     if nr1 and nr2:
 280                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
 281                     else:
 282                         tbl_item.append({"data": None})
 283                 except (IndexError, ValueError, TypeError):
 284                     logging.error("No data for {0}".format(tbl_item[0]["data"]))
 285                     tbl_item.append({"data": None})
 286                     continue
 287             else:
 288                 logging.error("Not supported command {0}. Skipping the table.".
 289                               format(cmd))
 290                 return None
 291         tbl_lst.append(tbl_item)
 292
 293     # Sort the table according to the relative change
 294     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
 295
 296     # Create the tables and write them to the files
 297     file_names = [
 298         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
 299         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
 300         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
 301         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
 302     ]
 303
 304     for file_name in file_names:
 305         logging.info("    Writing the file '{0}'".format(file_name))
 306         with open(file_name, "w") as file_handler:
 307             file_handler.write(",".join(header) + "\n")
 308             for item in tbl_lst:
 309                 if isinstance(item[-1]["data"], float):
 310                     rel_change = round(item[-1]["data"], 1)
 311                 else:
 312                     rel_change = item[-1]["data"]
 313                 if "ndr_top" in file_name \
 314                         and "ndr" in item[0]["data"] \
 315                         and rel_change >= 10.0:
 316                     _write_line_to_file(file_handler, item)
 317                 elif "pdr_top" in file_name \
 318                         and "pdr" in item[0]["data"] \
 319                         and rel_change >= 10.0:
 320                     _write_line_to_file(file_handler, item)
 321                 elif "ndr_low" in file_name \
 322                         and "ndr" in item[0]["data"] \
 323                         and rel_change < 10.0:
 324                     _write_line_to_file(file_handler, item)
 325                 elif "pdr_low" in file_name \
 326                         and "pdr" in item[0]["data"] \
 327                         and rel_change < 10.0:
 328                     _write_line_to_file(file_handler, item)
 329
 330     logging.info("  Done.")
 331
 332
 333 def _read_csv_template(file_name):
 334     """Read the template from a .csv file.
 335
 336     :param file_name: Name / full path / relative path of the file to read.
 337     :type file_name: str
 338     :returns: Data from the template as list (lines) of lists (items on line).
 339     :rtype: list
 340     :raises: PresentationError if it is not possible to read the file.
 341     """
 342
 343     try:
 344         with open(file_name, 'r') as csv_file:
 345             tmpl_data = list()
 346             for line in csv_file:
 347                 tmpl_data.append(line[:-1].split(","))
 348         return tmpl_data
 349     except IOError as err:
 350         raise PresentationError(str(err), level="ERROR")
 351
 352
 353 def table_performance_comparison(table, input_data):
 354     """Generate the table(s) with algorithm: table_performance_comparison
 355     specified in the specification file.
 356
 357     :param table: Table to generate.
 358     :param input_data: Data to process.
 359     :type table: pandas.Series
 360     :type input_data: InputData
 361     """
 362
 363     logging.info("  Generating the table {0} ...".
 364                  format(table.get("title", "")))
 365
 366     # Transform the data
 367     logging.info("    Creating the data set for the {0} '{1}'.".
 368                  format(table.get("type", ""), table.get("title", "")))
 369     data = input_data.filter_data(table, continue_on_error=True)
 370
 371     # Prepare the header of the tables
 372     try:
 373         header = ["Test case", ]
 374
 375         history = table.get("history", None)
 376         if history:
 377             for item in history:
 378                 header.extend(
 379                     ["{0} Throughput [Mpps]".format(item["title"]),
 380                      "{0} Stdev [Mpps]".format(item["title"])])
 381         header.extend(
 382             ["{0} Throughput [Mpps]".format(table["reference"]["title"]),
 383              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 384              "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 385              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 386              "Change [%]"])
 387         header_str = ",".join(header) + "\n"
 388     except (AttributeError, KeyError) as err:
 389         logging.error("The model is invalid, missing parameter: {0}".
 390                       format(err))
 391         return
 392
 393     # Prepare data to the table:
 394     tbl_dict = dict()
 395     for job, builds in table["reference"]["data"].items():
 396         for build in builds:
 397             for tst_name, tst_data in data[job][str(build)].iteritems():
 398                 if tbl_dict.get(tst_name, None) is None:
 399                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 400                                             "-".join(tst_data["name"].
 401                                                      split("-")[1:]))
 402                     tbl_dict[tst_name] = {"name": name,
 403                                           "ref-data": list(),
 404                                           "cmp-data": list()}
 405                 try:
 406                     tbl_dict[tst_name]["ref-data"].\
 407                         append(tst_data["throughput"]["value"])
 408                 except TypeError:
 409                     pass  # No data in output.xml for this test
 410
 411     for job, builds in table["compare"]["data"].items():
 412         for build in builds:
 413             for tst_name, tst_data in data[job][str(build)].iteritems():
 414                 try:
 415                     tbl_dict[tst_name]["cmp-data"].\
 416                         append(tst_data["throughput"]["value"])
 417                 except KeyError:
 418                     pass
 419                 except TypeError:
 420                     tbl_dict.pop(tst_name, None)
 421     if history:
 422         for item in history:
 423             for job, builds in item["data"].items():
 424                 for build in builds:
 425                     for tst_name, tst_data in data[job][str(build)].iteritems():
 426                         if tbl_dict.get(tst_name, None) is None:
 427                             continue
 428                         if tbl_dict[tst_name].get("history", None) is None:
 429                             tbl_dict[tst_name]["history"] = OrderedDict()
 430                         if tbl_dict[tst_name]["history"].get(item["title"],
 431                                                              None) is None:
 432                             tbl_dict[tst_name]["history"][item["title"]] = \
 433                                 list()
 434                         try:
 435                             tbl_dict[tst_name]["history"][item["title"]].\
 436                                 append(tst_data["throughput"]["value"])
 437                         except (TypeError, KeyError):
 438                             pass
 439
 440     tbl_lst = list()
 441     for tst_name in tbl_dict.keys():
 442         item = [tbl_dict[tst_name]["name"], ]
 443         if history:
 444             if tbl_dict[tst_name].get("history", None) is not None:
 445                 for hist_data in tbl_dict[tst_name]["history"].values():
 446                     if hist_data:
 447                         item.append(round(mean(hist_data) / 1000000, 2))
 448                         item.append(round(stdev(hist_data) / 1000000, 2))
 449                     else:
 450                         item.extend([None, None])
 451             else:
 452                 item.extend([None, None])
 453         data_t = tbl_dict[tst_name]["ref-data"]
 454         if data_t:
 455             item.append(round(mean(data_t) / 1000000, 2))
 456             item.append(round(stdev(data_t) / 1000000, 2))
 457         else:
 458             item.extend([None, None])
 459         data_t = tbl_dict[tst_name]["cmp-data"]
 460         if data_t:
 461             item.append(round(mean(data_t) / 1000000, 2))
 462             item.append(round(stdev(data_t) / 1000000, 2))
 463         else:
 464             item.extend([None, None])
 465         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
 466             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 467         if len(item) == len(header):
 468             tbl_lst.append(item)
 469
 470     # Sort the table according to the relative change
 471     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 472
 473     # Generate tables:
 474     # All tests in csv:
 475     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
 476                                                table["output-file-ext"]),
 477                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
 478                                                table["output-file-ext"]),
 479                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
 480                                                table["output-file-ext"]),
 481                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 482                                                table["output-file-ext"]),
 483                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
 484                                                table["output-file-ext"]),
 485                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
 486                                                table["output-file-ext"])
 487                  ]
 488     for file_name in tbl_names:
 489         logging.info("      Writing file: '{0}'".format(file_name))
 490         with open(file_name, "w") as file_handler:
 491             file_handler.write(header_str)
 492             for test in tbl_lst:
 493                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
 494                         file_name.split("-")[-2] in test[0]):  # cores
 495                     test[0] = "-".join(test[0].split("-")[:-1])
 496                     file_handler.write(",".join([str(item) for item in test]) +
 497                                        "\n")
 498
 499     # All tests in txt:
 500     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
 501                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
 502                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
 503                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
 504                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
 505                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
 506                      ]
 507
 508     for i, txt_name in enumerate(tbl_names_txt):
 509         txt_table = None
 510         logging.info("      Writing file: '{0}'".format(txt_name))
 511         with open(tbl_names[i], 'rb') as csv_file:
 512             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 513             for row in csv_content:
 514                 if txt_table is None:
 515                     txt_table = prettytable.PrettyTable(row)
 516                 else:
 517                     txt_table.add_row(row)
 518             txt_table.align["Test case"] = "l"
 519         with open(txt_name, "w") as txt_file:
 520             txt_file.write(str(txt_table))
 521
 522     # Selected tests in csv:
 523     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
 524                                                table["output-file-ext"])
 525     with open(input_file, "r") as in_file:
 526         lines = list()
 527         for line in in_file:
 528             lines.append(line)
 529
 530     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
 531                                                table["output-file-ext"])
 532     logging.info("      Writing file: '{0}'".format(output_file))
 533     with open(output_file, "w") as out_file:
 534         out_file.write(header_str)
 535         for i, line in enumerate(lines[1:]):
 536             if i == table["nr-of-tests-shown"]:
 537                 break
 538             out_file.write(line)
 539
 540     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
 541                                                   table["output-file-ext"])
 542     logging.info("      Writing file: '{0}'".format(output_file))
 543     with open(output_file, "w") as out_file:
 544         out_file.write(header_str)
 545         for i, line in enumerate(lines[-1:0:-1]):
 546             if i == table["nr-of-tests-shown"]:
 547                 break
 548             out_file.write(line)
 549
 550     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 551                                                table["output-file-ext"])
 552     with open(input_file, "r") as in_file:
 553         lines = list()
 554         for line in in_file:
 555             lines.append(line)
 556
 557     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
 558                                                table["output-file-ext"])
 559     logging.info("      Writing file: '{0}'".format(output_file))
 560     with open(output_file, "w") as out_file:
 561         out_file.write(header_str)
 562         for i, line in enumerate(lines[1:]):
 563             if i == table["nr-of-tests-shown"]:
 564                 break
 565             out_file.write(line)
 566
 567     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
 568                                                   table["output-file-ext"])
 569     logging.info("      Writing file: '{0}'".format(output_file))
 570     with open(output_file, "w") as out_file:
 571         out_file.write(header_str)
 572         for i, line in enumerate(lines[-1:0:-1]):
 573             if i == table["nr-of-tests-shown"]:
 574                 break
 575             out_file.write(line)
 576
 577
 578 def table_performance_comparison_mrr(table, input_data):
 579     """Generate the table(s) with algorithm: table_performance_comparison_mrr
 580     specified in the specification file.
 581
 582     :param table: Table to generate.
 583     :param input_data: Data to process.
 584     :type table: pandas.Series
 585     :type input_data: InputData
 586     """
 587
 588     logging.info("  Generating the table {0} ...".
 589                  format(table.get("title", "")))
 590
 591     # Transform the data
 592     logging.info("    Creating the data set for the {0} '{1}'.".
 593                  format(table.get("type", ""), table.get("title", "")))
 594     data = input_data.filter_data(table, continue_on_error=True)
 595
 596     # Prepare the header of the tables
 597     try:
 598         header = ["Test case",
 599                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 600                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 601                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 602                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 603                   "Change [%]"]
 604         header_str = ",".join(header) + "\n"
 605     except (AttributeError, KeyError) as err:
 606         logging.error("The model is invalid, missing parameter: {0}".
 607                       format(err))
 608         return
 609
 610     # Prepare data to the table:
 611     tbl_dict = dict()
 612     for job, builds in table["reference"]["data"].items():
 613         for build in builds:
 614             for tst_name, tst_data in data[job][str(build)].iteritems():
 615                 if tbl_dict.get(tst_name, None) is None:
 616                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 617                                             "-".join(tst_data["name"].
 618                                                      split("-")[1:]))
 619                     tbl_dict[tst_name] = {"name": name,
 620                                           "ref-data": list(),
 621                                           "cmp-data": list()}
 622                 try:
 623                     tbl_dict[tst_name]["ref-data"].\
 624                         append(tst_data["result"]["throughput"])
 625                 except TypeError:
 626                     pass  # No data in output.xml for this test
 627
 628     for job, builds in table["compare"]["data"].items():
 629         for build in builds:
 630             for tst_name, tst_data in data[job][str(build)].iteritems():
 631                 try:
 632                     tbl_dict[tst_name]["cmp-data"].\
 633                         append(tst_data["result"]["throughput"])
 634                 except KeyError:
 635                     pass
 636                 except TypeError:
 637                     tbl_dict.pop(tst_name, None)
 638
 639     tbl_lst = list()
 640     for tst_name in tbl_dict.keys():
 641         item = [tbl_dict[tst_name]["name"], ]
 642         data_t = tbl_dict[tst_name]["ref-data"]
 643         if data_t:
 644             item.append(round(mean(data_t) / 1000000, 2))
 645             item.append(round(stdev(data_t) / 1000000, 2))
 646         else:
 647             item.extend([None, None])
 648         data_t = tbl_dict[tst_name]["cmp-data"]
 649         if data_t:
 650             item.append(round(mean(data_t) / 1000000, 2))
 651             item.append(round(stdev(data_t) / 1000000, 2))
 652         else:
 653             item.extend([None, None])
 654         if item[1] is not None and item[3] is not None and item[1] != 0:
 655             item.append(int(relative_change(float(item[1]), float(item[3]))))
 656         if len(item) == 6:
 657             tbl_lst.append(item)
 658
 659     # Sort the table according to the relative change
 660     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 661
 662     # Generate tables:
 663     # All tests in csv:
 664     tbl_names = ["{0}-1t1c-full{1}".format(table["output-file"],
 665                                            table["output-file-ext"]),
 666                  "{0}-2t2c-full{1}".format(table["output-file"],
 667                                            table["output-file-ext"]),
 668                  "{0}-4t4c-full{1}".format(table["output-file"],
 669                                            table["output-file-ext"])
 670                  ]
 671     for file_name in tbl_names:
 672         logging.info("      Writing file: '{0}'".format(file_name))
 673         with open(file_name, "w") as file_handler:
 674             file_handler.write(header_str)
 675             for test in tbl_lst:
 676                 if file_name.split("-")[-2] in test[0]:  # cores
 677                     test[0] = "-".join(test[0].split("-")[:-1])
 678                     file_handler.write(",".join([str(item) for item in test]) +
 679                                        "\n")
 680
 681     # All tests in txt:
 682     tbl_names_txt = ["{0}-1t1c-full.txt".format(table["output-file"]),
 683                      "{0}-2t2c-full.txt".format(table["output-file"]),
 684                      "{0}-4t4c-full.txt".format(table["output-file"])
 685                      ]
 686
 687     for i, txt_name in enumerate(tbl_names_txt):
 688         txt_table = None
 689         logging.info("      Writing file: '{0}'".format(txt_name))
 690         with open(tbl_names[i], 'rb') as csv_file:
 691             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 692             for row in csv_content:
 693                 if txt_table is None:
 694                     txt_table = prettytable.PrettyTable(row)
 695                 else:
 696                     txt_table.add_row(row)
 697             txt_table.align["Test case"] = "l"
 698         with open(txt_name, "w") as txt_file:
 699             txt_file.write(str(txt_table))
 700
 701
 702 def table_performance_trending_dashboard(table, input_data):
 703     """Generate the table(s) with algorithm: table_performance_comparison
 704     specified in the specification file.
 705
 706     :param table: Table to generate.
 707     :param input_data: Data to process.
 708     :type table: pandas.Series
 709     :type input_data: InputData
 710     """
 711
 712     logging.info("  Generating the table {0} ...".
 713                  format(table.get("title", "")))
 714
 715     # Transform the data
 716     logging.info("    Creating the data set for the {0} '{1}'.".
 717                  format(table.get("type", ""), table.get("title", "")))
 718     data = input_data.filter_data(table, continue_on_error=True)
 719
 720     # Prepare the header of the tables
 721     header = ["Test Case",
 722               "Trend [Mpps]",
 723               "Short-Term Change [%]",
 724               "Long-Term Change [%]",
 725               "Regressions [#]",
 726               "Progressions [#]",
 727               "Outliers [#]"
 728               ]
 729     header_str = ",".join(header) + "\n"
 730
 731     # Prepare data to the table:
 732     tbl_dict = dict()
 733     for job, builds in table["data"].items():
 734         for build in builds:
 735             for tst_name, tst_data in data[job][str(build)].iteritems():
 736                 if tst_name.lower() in table["ignore-list"]:
 737                     continue
 738                 if tbl_dict.get(tst_name, None) is None:
 739                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 740                                             "-".join(tst_data["name"].
 741                                                      split("-")[1:]))
 742                     tbl_dict[tst_name] = {"name": name,
 743                                           "data": OrderedDict()}
 744                 try:
 745                     tbl_dict[tst_name]["data"][str(build)] =  \
 746                         tst_data["result"]["throughput"]
 747                 except (TypeError, KeyError):
 748                     pass  # No data in output.xml for this test
 749
 750     tbl_lst = list()
 751     for tst_name in tbl_dict.keys():
 752         if len(tbl_dict[tst_name]["data"]) < 3:
 753             continue
 754
 755         data_t = pd.Series(tbl_dict[tst_name]["data"])
 756         last_key = data_t.keys()[-1]
 757         win_size = min(data_t.size, table["window"])
 758         win_first_idx = data_t.size - win_size
 759         key_14 = data_t.keys()[win_first_idx]
 760         long_win_size = min(data_t.size, table["long-trend-window"])
 761         median_t = data_t.rolling(window=win_size, min_periods=2).median()
 762         median_first_idx = median_t.size - long_win_size
 763         try:
 764             max_median = max(
 765                 [x for x in median_t.values[median_first_idx:-win_size]
 766                  if not isnan(x)])
 767         except ValueError:
 768             max_median = nan
 769         try:
 770             last_median_t = median_t[last_key]
 771         except KeyError:
 772             last_median_t = nan
 773         try:
 774             median_t_14 = median_t[key_14]
 775         except KeyError:
 776             median_t_14 = nan
 777
 778         if isnan(last_median_t) or isnan(median_t_14) or median_t_14 == 0.0:
 779             rel_change_last = nan
 780         else:
 781             rel_change_last = round(
 782                 ((last_median_t - median_t_14) / median_t_14) * 100, 2)
 783
 784         if isnan(max_median) or isnan(last_median_t) or max_median == 0.0:
 785             rel_change_long = nan
 786         else:
 787             rel_change_long = round(
 788                 ((last_median_t - max_median) / max_median) * 100, 2)
 789
 790         # Classification list:
 791         classification_lst, _ = classify_anomalies(data_t)
 792
 793         if classification_lst:
 794             if isnan(rel_change_last) and isnan(rel_change_long):
 795                 continue
 796             tbl_lst.append(
 797                 [tbl_dict[tst_name]["name"],
 798                  '-' if isnan(last_median_t) else
 799                  round(last_median_t / 1000000, 2),
 800                  '-' if isnan(rel_change_last) else rel_change_last,
 801                  '-' if isnan(rel_change_long) else rel_change_long,
 802                  classification_lst[win_first_idx:].count("regression"),
 803                  classification_lst[win_first_idx:].count("progression"),
 804                  classification_lst[win_first_idx:].count("outlier")])
 805
 806     tbl_lst.sort(key=lambda rel: rel[0])
 807
 808     tbl_sorted = list()
 809     for nrr in range(table["window"], -1, -1):
 810         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
 811         for nrp in range(table["window"], -1, -1):
 812             tbl_pro = [item for item in tbl_reg if item[5] == nrp]
 813             for nro in range(table["window"], -1, -1):
 814                 tbl_out = [item for item in tbl_pro if item[6] == nro]
 815                 tbl_out.sort(key=lambda rel: rel[2])
 816                 tbl_sorted.extend(tbl_out)
 817
 818     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
 819
 820     logging.info("    Writing file: '{0}'".format(file_name))
 821     with open(file_name, "w") as file_handler:
 822         file_handler.write(header_str)
 823         for test in tbl_sorted:
 824             file_handler.write(",".join([str(item) for item in test]) + '\n')
 825
 826     txt_file_name = "{0}.txt".format(table["output-file"])
 827     txt_table = None
 828     logging.info("    Writing file: '{0}'".format(txt_file_name))
 829     with open(file_name, 'rb') as csv_file:
 830         csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 831         for row in csv_content:
 832             if txt_table is None:
 833                 txt_table = prettytable.PrettyTable(row)
 834             else:
 835                 txt_table.add_row(row)
 836         txt_table.align["Test case"] = "l"
 837     with open(txt_file_name, "w") as txt_file:
 838         txt_file.write(str(txt_table))
 839
 840
 841 def table_performance_trending_dashboard_html(table, input_data):
 842     """Generate the table(s) with algorithm:
 843     table_performance_trending_dashboard_html specified in the specification
 844     file.
 845
 846     :param table: Table to generate.
 847     :param input_data: Data to process.
 848     :type table: pandas.Series
 849     :type input_data: InputData
 850     """
 851
 852     logging.info("  Generating the table {0} ...".
 853                  format(table.get("title", "")))
 854
 855     try:
 856         with open(table["input-file"], 'rb') as csv_file:
 857             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 858             csv_lst = [item for item in csv_content]
 859     except KeyError:
 860         logging.warning("The input file is not defined.")
 861         return
 862     except csv.Error as err:
 863         logging.warning("Not possible to process the file '{0}'.\n{1}".
 864                         format(table["input-file"], err))
 865         return
 866
 867     # Table:
 868     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
 869
 870     # Table header:
 871     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
 872     for idx, item in enumerate(csv_lst[0]):
 873         alignment = "left" if idx == 0 else "center"
 874         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
 875         th.text = item
 876
 877     # Rows:
 878     colors = {"regression": ("#ffcccc", "#ff9999"),
 879               "progression": ("#c6ecc6", "#9fdf9f"),
 880               "outlier": ("#e6e6e6", "#cccccc"),
 881               "normal": ("#e9f1fb", "#d4e4f7")}
 882     for r_idx, row in enumerate(csv_lst[1:]):
 883         if int(row[4]):
 884             color = "regression"
 885         elif int(row[5]):
 886             color = "progression"
 887         elif int(row[6]):
 888             color = "outlier"
 889         else:
 890             color = "normal"
 891         background = colors[color][r_idx % 2]
 892         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
 893
 894         # Columns:
 895         for c_idx, item in enumerate(row):
 896             alignment = "left" if c_idx == 0 else "center"
 897             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
 898             # Name:
 899             url = "../trending/"
 900             file_name = ""
 901             anchor = "#"
 902             feature = ""
 903             if c_idx == 0:
 904                 if "lbdpdk" in item or "lbvpp" in item:
 905                     file_name = "link_bonding.html"
 906
 907                 elif "testpmd" in item or "l3fwd" in item:
 908                     file_name = "dpdk.html"
 909
 910                 elif "memif" in item:
 911                     file_name = "container_memif.html"
 912
 913                 elif "srv6" in item:
 914                     file_name = "srv6.html"
 915
 916                 elif "vhost" in item:
 917                     if "l2xcbase" in item or "l2bdbasemaclrn" in item:
 918                         file_name = "vm_vhost_l2.html"
 919                     elif "ip4base" in item:
 920                         file_name = "vm_vhost_ip4.html"
 921
 922                 elif "ipsec" in item:
 923                     file_name = "ipsec.html"
 924
 925                 elif "ethip4lispip" in item or "ethip4vxlan" in item:
 926                     file_name = "ip4_tunnels.html"
 927
 928                 elif "ip4base" in item or "ip4scale" in item:
 929                     file_name = "ip4.html"
 930                     if "iacl" in item or "snat" in item or "cop" in item:
 931                         feature = "-features"
 932
 933                 elif "ip6base" in item or "ip6scale" in item:
 934                     file_name = "ip6.html"
 935
 936                 elif "l2xcbase" in item or "l2xcscale" in item \
 937                         or "l2bdbasemaclrn" in item or "l2bdscale" in item \
 938                         or "l2dbbasemaclrn" in item or "l2dbscale" in item:
 939                     file_name = "l2.html"
 940                     if "iacl" in item:
 941                         feature = "-features"
 942
 943                 if "x520" in item:
 944                     anchor += "x520-"
 945                 elif "x710" in item:
 946                     anchor += "x710-"
 947                 elif "xl710" in item:
 948                     anchor += "xl710-"
 949
 950                 if "64b" in item:
 951                     anchor += "64b-"
 952                 elif "78b" in item:
 953                     anchor += "78b-"
 954                 elif "imix" in item:
 955                     anchor += "imix-"
 956                 elif "9000b" in item:
 957                     anchor += "9000b-"
 958                 elif "1518" in item:
 959                     anchor += "1518b-"
 960
 961                 if "1t1c" in item:
 962                     anchor += "1t1c"
 963                 elif "2t2c" in item:
 964                     anchor += "2t2c"
 965                 elif "4t4c" in item:
 966                     anchor += "4t4c"
 967
 968                 url = url + file_name + anchor + feature
 969
 970                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
 971                 ref.text = item
 972
 973             else:
 974                 td.text = item
 975
 976     try:
 977         with open(table["output-file"], 'w') as html_file:
 978             logging.info("    Writing file: '{0}'".format(table["output-file"]))
 979             html_file.write(".. raw:: html\n\n\t")
 980             html_file.write(ET.tostring(dashboard))
 981             html_file.write("\n\t<p><br><br></p>\n")
 982     except KeyError:
 983         logging.warning("The output file is not defined.")
 984         return