resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import prettytable
  21 import pandas as pd
  22
  23 from string import replace
  24 from collections import OrderedDict
  25 from numpy import nan, isnan
  26 from xml.etree import ElementTree as ET
  27
  28 from errors import PresentationError
  29 from utils import mean, stdev, relative_change, remove_outliers, split_outliers
  30
  31
  32 def generate_tables(spec, data):
  33     """Generate all tables specified in the specification file.
  34
  35     :param spec: Specification read from the specification file.
  36     :param data: Data to process.
  37     :type spec: Specification
  38     :type data: InputData
  39     """
  40
  41     logging.info("Generating the tables ...")
  42     for table in spec.tables:
  43         try:
  44             eval(table["algorithm"])(table, data)
  45         except NameError:
  46             logging.error("The algorithm '{0}' is not defined.".
  47                           format(table["algorithm"]))
  48     logging.info("Done.")
  49
  50
  51 def table_details(table, input_data):
  52     """Generate the table(s) with algorithm: table_detailed_test_results
  53     specified in the specification file.
  54
  55     :param table: Table to generate.
  56     :param input_data: Data to process.
  57     :type table: pandas.Series
  58     :type input_data: InputData
  59     """
  60
  61     logging.info("  Generating the table {0} ...".
  62                  format(table.get("title", "")))
  63
  64     # Transform the data
  65     data = input_data.filter_data(table)
  66
  67     # Prepare the header of the tables
  68     header = list()
  69     for column in table["columns"]:
  70         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  71
  72     # Generate the data for the table according to the model in the table
  73     # specification
  74     job = table["data"].keys()[0]
  75     build = str(table["data"][job][0])
  76     try:
  77         suites = input_data.suites(job, build)
  78     except KeyError:
  79         logging.error("    No data available. The table will not be generated.")
  80         return
  81
  82     for suite_longname, suite in suites.iteritems():
  83         # Generate data
  84         suite_name = suite["name"]
  85         table_lst = list()
  86         for test in data[job][build].keys():
  87             if data[job][build][test]["parent"] in suite_name:
  88                 row_lst = list()
  89                 for column in table["columns"]:
  90                     try:
  91                         col_data = str(data[job][build][test][column["data"].
  92                                        split(" ")[1]]).replace('"', '""')
  93                         if column["data"].split(" ")[1] in ("vat-history",
  94                                                             "show-run"):
  95                             col_data = replace(col_data, " |br| ", "",
  96                                                maxreplace=1)
  97                             col_data = " |prein| {0} |preout| ".\
  98                                 format(col_data[:-5])
  99                         row_lst.append('"{0}"'.format(col_data))
 100                     except KeyError:
 101                         row_lst.append("No data")
 102                 table_lst.append(row_lst)
 103
 104         # Write the data to file
 105         if table_lst:
 106             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 107                                             table["output-file-ext"])
 108             logging.info("      Writing file: '{}'".format(file_name))
 109             with open(file_name, "w") as file_handler:
 110                 file_handler.write(",".join(header) + "\n")
 111                 for item in table_lst:
 112                     file_handler.write(",".join(item) + "\n")
 113
 114     logging.info("  Done.")
 115
 116
 117 def table_merged_details(table, input_data):
 118     """Generate the table(s) with algorithm: table_merged_details
 119     specified in the specification file.
 120
 121     :param table: Table to generate.
 122     :param input_data: Data to process.
 123     :type table: pandas.Series
 124     :type input_data: InputData
 125     """
 126
 127     logging.info("  Generating the table {0} ...".
 128                  format(table.get("title", "")))
 129
 130     # Transform the data
 131     data = input_data.filter_data(table)
 132     data = input_data.merge_data(data)
 133     data.sort_index(inplace=True)
 134
 135     suites = input_data.filter_data(table, data_set="suites")
 136     suites = input_data.merge_data(suites)
 137
 138     # Prepare the header of the tables
 139     header = list()
 140     for column in table["columns"]:
 141         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 142
 143     for _, suite in suites.iteritems():
 144         # Generate data
 145         suite_name = suite["name"]
 146         table_lst = list()
 147         for test in data.keys():
 148             if data[test]["parent"] in suite_name:
 149                 row_lst = list()
 150                 for column in table["columns"]:
 151                     try:
 152                         col_data = str(data[test][column["data"].
 153                                        split(" ")[1]]).replace('"', '""')
 154                         if column["data"].split(" ")[1] in ("vat-history",
 155                                                             "show-run"):
 156                             col_data = replace(col_data, " |br| ", "",
 157                                                maxreplace=1)
 158                             col_data = " |prein| {0} |preout| ".\
 159                                 format(col_data[:-5])
 160                         row_lst.append('"{0}"'.format(col_data))
 161                     except KeyError:
 162                         row_lst.append("No data")
 163                 table_lst.append(row_lst)
 164
 165         # Write the data to file
 166         if table_lst:
 167             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 168                                             table["output-file-ext"])
 169             logging.info("      Writing file: '{}'".format(file_name))
 170             with open(file_name, "w") as file_handler:
 171                 file_handler.write(",".join(header) + "\n")
 172                 for item in table_lst:
 173                     file_handler.write(",".join(item) + "\n")
 174
 175     logging.info("  Done.")
 176
 177
 178 def table_performance_improvements(table, input_data):
 179     """Generate the table(s) with algorithm: table_performance_improvements
 180     specified in the specification file.
 181
 182     :param table: Table to generate.
 183     :param input_data: Data to process.
 184     :type table: pandas.Series
 185     :type input_data: InputData
 186     """
 187
 188     def _write_line_to_file(file_handler, data):
 189         """Write a line to the .csv file.
 190
 191         :param file_handler: File handler for the csv file. It must be open for
 192          writing text.
 193         :param data: Item to be written to the file.
 194         :type file_handler: BinaryIO
 195         :type data: list
 196         """
 197
 198         line_lst = list()
 199         for item in data:
 200             if isinstance(item["data"], str):
 201                 # Remove -?drdisc from the end
 202                 if item["data"].endswith("drdisc"):
 203                     item["data"] = item["data"][:-8]
 204                 line_lst.append(item["data"])
 205             elif isinstance(item["data"], float):
 206                 line_lst.append("{:.1f}".format(item["data"]))
 207             elif item["data"] is None:
 208                 line_lst.append("")
 209         file_handler.write(",".join(line_lst) + "\n")
 210
 211     logging.info("  Generating the table {0} ...".
 212                  format(table.get("title", "")))
 213
 214     # Read the template
 215     file_name = table.get("template", None)
 216     if file_name:
 217         try:
 218             tmpl = _read_csv_template(file_name)
 219         except PresentationError:
 220             logging.error("  The template '{0}' does not exist. Skipping the "
 221                           "table.".format(file_name))
 222             return None
 223     else:
 224         logging.error("The template is not defined. Skipping the table.")
 225         return None
 226
 227     # Transform the data
 228     data = input_data.filter_data(table)
 229
 230     # Prepare the header of the tables
 231     header = list()
 232     for column in table["columns"]:
 233         header.append(column["title"])
 234
 235     # Generate the data for the table according to the model in the table
 236     # specification
 237     tbl_lst = list()
 238     for tmpl_item in tmpl:
 239         tbl_item = list()
 240         for column in table["columns"]:
 241             cmd = column["data"].split(" ")[0]
 242             args = column["data"].split(" ")[1:]
 243             if cmd == "template":
 244                 try:
 245                     val = float(tmpl_item[int(args[0])])
 246                 except ValueError:
 247                     val = tmpl_item[int(args[0])]
 248                 tbl_item.append({"data": val})
 249             elif cmd == "data":
 250                 jobs = args[0:-1]
 251                 operation = args[-1]
 252                 data_lst = list()
 253                 for job in jobs:
 254                     for build in data[job]:
 255                         try:
 256                             data_lst.append(float(build[tmpl_item[0]]
 257                                                   ["throughput"]["value"]))
 258                         except (KeyError, TypeError):
 259                             # No data, ignore
 260                             continue
 261                 if data_lst:
 262                     tbl_item.append({"data": (eval(operation)(data_lst)) /
 263                                              1000000})
 264                 else:
 265                     tbl_item.append({"data": None})
 266             elif cmd == "operation":
 267                 operation = args[0]
 268                 try:
 269                     nr1 = float(tbl_item[int(args[1])]["data"])
 270                     nr2 = float(tbl_item[int(args[2])]["data"])
 271                     if nr1 and nr2:
 272                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
 273                     else:
 274                         tbl_item.append({"data": None})
 275                 except (IndexError, ValueError, TypeError):
 276                     logging.error("No data for {0}".format(tbl_item[0]["data"]))
 277                     tbl_item.append({"data": None})
 278                     continue
 279             else:
 280                 logging.error("Not supported command {0}. Skipping the table.".
 281                               format(cmd))
 282                 return None
 283         tbl_lst.append(tbl_item)
 284
 285     # Sort the table according to the relative change
 286     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
 287
 288     # Create the tables and write them to the files
 289     file_names = [
 290         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
 291         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
 292         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
 293         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
 294     ]
 295
 296     for file_name in file_names:
 297         logging.info("    Writing the file '{0}'".format(file_name))
 298         with open(file_name, "w") as file_handler:
 299             file_handler.write(",".join(header) + "\n")
 300             for item in tbl_lst:
 301                 if isinstance(item[-1]["data"], float):
 302                     rel_change = round(item[-1]["data"], 1)
 303                 else:
 304                     rel_change = item[-1]["data"]
 305                 if "ndr_top" in file_name \
 306                         and "ndr" in item[0]["data"] \
 307                         and rel_change >= 10.0:
 308                     _write_line_to_file(file_handler, item)
 309                 elif "pdr_top" in file_name \
 310                         and "pdr" in item[0]["data"] \
 311                         and rel_change >= 10.0:
 312                     _write_line_to_file(file_handler, item)
 313                 elif "ndr_low" in file_name \
 314                         and "ndr" in item[0]["data"] \
 315                         and rel_change < 10.0:
 316                     _write_line_to_file(file_handler, item)
 317                 elif "pdr_low" in file_name \
 318                         and "pdr" in item[0]["data"] \
 319                         and rel_change < 10.0:
 320                     _write_line_to_file(file_handler, item)
 321
 322     logging.info("  Done.")
 323
 324
 325 def _read_csv_template(file_name):
 326     """Read the template from a .csv file.
 327
 328     :param file_name: Name / full path / relative path of the file to read.
 329     :type file_name: str
 330     :returns: Data from the template as list (lines) of lists (items on line).
 331     :rtype: list
 332     :raises: PresentationError if it is not possible to read the file.
 333     """
 334
 335     try:
 336         with open(file_name, 'r') as csv_file:
 337             tmpl_data = list()
 338             for line in csv_file:
 339                 tmpl_data.append(line[:-1].split(","))
 340         return tmpl_data
 341     except IOError as err:
 342         raise PresentationError(str(err), level="ERROR")
 343
 344
 345 def table_performance_comparison(table, input_data):
 346     """Generate the table(s) with algorithm: table_performance_comparison
 347     specified in the specification file.
 348
 349     :param table: Table to generate.
 350     :param input_data: Data to process.
 351     :type table: pandas.Series
 352     :type input_data: InputData
 353     """
 354
 355     logging.info("  Generating the table {0} ...".
 356                  format(table.get("title", "")))
 357
 358     # Transform the data
 359     data = input_data.filter_data(table, continue_on_error=True)
 360
 361     # Prepare the header of the tables
 362     try:
 363         header = ["Test case", ]
 364
 365         history = table.get("history", None)
 366         if history:
 367             for item in history:
 368                 header.extend(
 369                     ["{0} Throughput [Mpps]".format(item["title"]),
 370                      "{0} Stdev [Mpps]".format(item["title"])])
 371         header.extend(
 372             ["{0} Throughput [Mpps]".format(table["reference"]["title"]),
 373              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 374              "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 375              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 376              "Change [%]"])
 377         header_str = ",".join(header) + "\n"
 378     except (AttributeError, KeyError) as err:
 379         logging.error("The model is invalid, missing parameter: {0}".
 380                       format(err))
 381         return
 382
 383     # Prepare data to the table:
 384     tbl_dict = dict()
 385     for job, builds in table["reference"]["data"].items():
 386         for build in builds:
 387             for tst_name, tst_data in data[job][str(build)].iteritems():
 388                 if tbl_dict.get(tst_name, None) is None:
 389                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 390                                             "-".join(tst_data["name"].
 391                                                      split("-")[1:]))
 392                     tbl_dict[tst_name] = {"name": name,
 393                                           "ref-data": list(),
 394                                           "cmp-data": list()}
 395                 try:
 396                     tbl_dict[tst_name]["ref-data"].\
 397                         append(tst_data["throughput"]["value"])
 398                 except TypeError:
 399                     pass  # No data in output.xml for this test
 400
 401     for job, builds in table["compare"]["data"].items():
 402         for build in builds:
 403             for tst_name, tst_data in data[job][str(build)].iteritems():
 404                 try:
 405                     tbl_dict[tst_name]["cmp-data"].\
 406                         append(tst_data["throughput"]["value"])
 407                 except KeyError:
 408                     pass
 409                 except TypeError:
 410                     tbl_dict.pop(tst_name, None)
 411     if history:
 412         for item in history:
 413             for job, builds in item["data"].items():
 414                 for build in builds:
 415                     for tst_name, tst_data in data[job][str(build)].iteritems():
 416                         if tbl_dict.get(tst_name, None) is None:
 417                             continue
 418                         if tbl_dict[tst_name].get("history", None) is None:
 419                             tbl_dict[tst_name]["history"] = OrderedDict()
 420                         if tbl_dict[tst_name]["history"].get(item["title"],
 421                                                              None) is None:
 422                             tbl_dict[tst_name]["history"][item["title"]] = \
 423                                 list()
 424                         try:
 425                             tbl_dict[tst_name]["history"][item["title"]].\
 426                                 append(tst_data["throughput"]["value"])
 427                         except (TypeError, KeyError):
 428                             pass
 429
 430     tbl_lst = list()
 431     for tst_name in tbl_dict.keys():
 432         item = [tbl_dict[tst_name]["name"], ]
 433         if history:
 434             if tbl_dict[tst_name].get("history", None) is not None:
 435                 for hist_data in tbl_dict[tst_name]["history"].values():
 436                     if hist_data:
 437                         data_t = remove_outliers(
 438                             hist_data, outlier_const=table["outlier-const"])
 439                         if data_t:
 440                             item.append(round(mean(data_t) / 1000000, 2))
 441                             item.append(round(stdev(data_t) / 1000000, 2))
 442                         else:
 443                             item.extend([None, None])
 444                     else:
 445                         item.extend([None, None])
 446             else:
 447                 item.extend([None, None])
 448         if tbl_dict[tst_name]["ref-data"]:
 449             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
 450                                      outlier_const=table["outlier-const"])
 451             # TODO: Specify window size.
 452             if data_t:
 453                 item.append(round(mean(data_t) / 1000000, 2))
 454                 item.append(round(stdev(data_t) / 1000000, 2))
 455             else:
 456                 item.extend([None, None])
 457         else:
 458             item.extend([None, None])
 459         if tbl_dict[tst_name]["cmp-data"]:
 460             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
 461                                      outlier_const=table["outlier-const"])
 462             # TODO: Specify window size.
 463             if data_t:
 464                 item.append(round(mean(data_t) / 1000000, 2))
 465                 item.append(round(stdev(data_t) / 1000000, 2))
 466             else:
 467                 item.extend([None, None])
 468         else:
 469             item.extend([None, None])
 470         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
 471             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 472         if len(item) == len(header):
 473             tbl_lst.append(item)
 474
 475     # Sort the table according to the relative change
 476     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 477
 478     # Generate tables:
 479     # All tests in csv:
 480     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
 481                                                table["output-file-ext"]),
 482                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
 483                                                table["output-file-ext"]),
 484                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
 485                                                table["output-file-ext"]),
 486                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 487                                                table["output-file-ext"]),
 488                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
 489                                                table["output-file-ext"]),
 490                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
 491                                                table["output-file-ext"])
 492                  ]
 493     for file_name in tbl_names:
 494         logging.info("      Writing file: '{0}'".format(file_name))
 495         with open(file_name, "w") as file_handler:
 496             file_handler.write(header_str)
 497             for test in tbl_lst:
 498                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
 499                         file_name.split("-")[-2] in test[0]):  # cores
 500                     test[0] = "-".join(test[0].split("-")[:-1])
 501                     file_handler.write(",".join([str(item) for item in test]) +
 502                                        "\n")
 503
 504     # All tests in txt:
 505     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
 506                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
 507                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
 508                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
 509                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
 510                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
 511                      ]
 512
 513     for i, txt_name in enumerate(tbl_names_txt):
 514         txt_table = None
 515         logging.info("      Writing file: '{0}'".format(txt_name))
 516         with open(tbl_names[i], 'rb') as csv_file:
 517             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 518             for row in csv_content:
 519                 if txt_table is None:
 520                     txt_table = prettytable.PrettyTable(row)
 521                 else:
 522                     txt_table.add_row(row)
 523             txt_table.align["Test case"] = "l"
 524         with open(txt_name, "w") as txt_file:
 525             txt_file.write(str(txt_table))
 526
 527     # Selected tests in csv:
 528     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
 529                                                table["output-file-ext"])
 530     with open(input_file, "r") as in_file:
 531         lines = list()
 532         for line in in_file:
 533             lines.append(line)
 534
 535     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
 536                                                table["output-file-ext"])
 537     logging.info("      Writing file: '{0}'".format(output_file))
 538     with open(output_file, "w") as out_file:
 539         out_file.write(header_str)
 540         for i, line in enumerate(lines[1:]):
 541             if i == table["nr-of-tests-shown"]:
 542                 break
 543             out_file.write(line)
 544
 545     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
 546                                                   table["output-file-ext"])
 547     logging.info("      Writing file: '{0}'".format(output_file))
 548     with open(output_file, "w") as out_file:
 549         out_file.write(header_str)
 550         for i, line in enumerate(lines[-1:0:-1]):
 551             if i == table["nr-of-tests-shown"]:
 552                 break
 553             out_file.write(line)
 554
 555     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 556                                                table["output-file-ext"])
 557     with open(input_file, "r") as in_file:
 558         lines = list()
 559         for line in in_file:
 560             lines.append(line)
 561
 562     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
 563                                                table["output-file-ext"])
 564     logging.info("      Writing file: '{0}'".format(output_file))
 565     with open(output_file, "w") as out_file:
 566         out_file.write(header_str)
 567         for i, line in enumerate(lines[1:]):
 568             if i == table["nr-of-tests-shown"]:
 569                 break
 570             out_file.write(line)
 571
 572     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
 573                                                   table["output-file-ext"])
 574     logging.info("      Writing file: '{0}'".format(output_file))
 575     with open(output_file, "w") as out_file:
 576         out_file.write(header_str)
 577         for i, line in enumerate(lines[-1:0:-1]):
 578             if i == table["nr-of-tests-shown"]:
 579                 break
 580             out_file.write(line)
 581
 582
 583 def table_performance_comparison_mrr(table, input_data):
 584     """Generate the table(s) with algorithm: table_performance_comparison_mrr
 585     specified in the specification file.
 586
 587     :param table: Table to generate.
 588     :param input_data: Data to process.
 589     :type table: pandas.Series
 590     :type input_data: InputData
 591     """
 592
 593     logging.info("  Generating the table {0} ...".
 594                  format(table.get("title", "")))
 595
 596     # Transform the data
 597     data = input_data.filter_data(table, continue_on_error=True)
 598
 599     # Prepare the header of the tables
 600     try:
 601         header = ["Test case",
 602                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 603                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 604                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 605                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 606                   "Change [%]"]
 607         header_str = ",".join(header) + "\n"
 608     except (AttributeError, KeyError) as err:
 609         logging.error("The model is invalid, missing parameter: {0}".
 610                       format(err))
 611         return
 612
 613     # Prepare data to the table:
 614     tbl_dict = dict()
 615     for job, builds in table["reference"]["data"].items():
 616         for build in builds:
 617             for tst_name, tst_data in data[job][str(build)].iteritems():
 618                 if tbl_dict.get(tst_name, None) is None:
 619                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 620                                             "-".join(tst_data["name"].
 621                                                      split("-")[1:]))
 622                     tbl_dict[tst_name] = {"name": name,
 623                                           "ref-data": list(),
 624                                           "cmp-data": list()}
 625                 try:
 626                     tbl_dict[tst_name]["ref-data"].\
 627                         append(tst_data["result"]["throughput"])
 628                 except TypeError:
 629                     pass  # No data in output.xml for this test
 630
 631     for job, builds in table["compare"]["data"].items():
 632         for build in builds:
 633             for tst_name, tst_data in data[job][str(build)].iteritems():
 634                 try:
 635                     tbl_dict[tst_name]["cmp-data"].\
 636                         append(tst_data["result"]["throughput"])
 637                 except KeyError:
 638                     pass
 639                 except TypeError:
 640                     tbl_dict.pop(tst_name, None)
 641
 642     tbl_lst = list()
 643     for tst_name in tbl_dict.keys():
 644         item = [tbl_dict[tst_name]["name"], ]
 645         if tbl_dict[tst_name]["ref-data"]:
 646             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
 647                                      outlier_const=table["outlier-const"])
 648             # TODO: Specify window size.
 649             if data_t:
 650                 item.append(round(mean(data_t) / 1000000, 2))
 651                 item.append(round(stdev(data_t) / 1000000, 2))
 652             else:
 653                 item.extend([None, None])
 654         else:
 655             item.extend([None, None])
 656         if tbl_dict[tst_name]["cmp-data"]:
 657             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
 658                                      outlier_const=table["outlier-const"])
 659             # TODO: Specify window size.
 660             if data_t:
 661                 item.append(round(mean(data_t) / 1000000, 2))
 662                 item.append(round(stdev(data_t) / 1000000, 2))
 663             else:
 664                 item.extend([None, None])
 665         else:
 666             item.extend([None, None])
 667         if item[1] is not None and item[3] is not None and item[1] != 0:
 668             item.append(int(relative_change(float(item[1]), float(item[3]))))
 669         if len(item) == 6:
 670             tbl_lst.append(item)
 671
 672     # Sort the table according to the relative change
 673     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 674
 675     # Generate tables:
 676     # All tests in csv:
 677     tbl_names = ["{0}-1t1c-full{1}".format(table["output-file"],
 678                                            table["output-file-ext"]),
 679                  "{0}-2t2c-full{1}".format(table["output-file"],
 680                                            table["output-file-ext"]),
 681                  "{0}-4t4c-full{1}".format(table["output-file"],
 682                                            table["output-file-ext"])
 683                  ]
 684     for file_name in tbl_names:
 685         logging.info("      Writing file: '{0}'".format(file_name))
 686         with open(file_name, "w") as file_handler:
 687             file_handler.write(header_str)
 688             for test in tbl_lst:
 689                 if file_name.split("-")[-2] in test[0]:  # cores
 690                     test[0] = "-".join(test[0].split("-")[:-1])
 691                     file_handler.write(",".join([str(item) for item in test]) +
 692                                        "\n")
 693
 694     # All tests in txt:
 695     tbl_names_txt = ["{0}-1t1c-full.txt".format(table["output-file"]),
 696                      "{0}-2t2c-full.txt".format(table["output-file"]),
 697                      "{0}-4t4c-full.txt".format(table["output-file"])
 698                      ]
 699
 700     for i, txt_name in enumerate(tbl_names_txt):
 701         txt_table = None
 702         logging.info("      Writing file: '{0}'".format(txt_name))
 703         with open(tbl_names[i], 'rb') as csv_file:
 704             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 705             for row in csv_content:
 706                 if txt_table is None:
 707                     txt_table = prettytable.PrettyTable(row)
 708                 else:
 709                     txt_table.add_row(row)
 710             txt_table.align["Test case"] = "l"
 711         with open(txt_name, "w") as txt_file:
 712             txt_file.write(str(txt_table))
 713
 714
 715 def table_performance_trending_dashboard(table, input_data):
 716     """Generate the table(s) with algorithm: table_performance_comparison
 717     specified in the specification file.
 718
 719     :param table: Table to generate.
 720     :param input_data: Data to process.
 721     :type table: pandas.Series
 722     :type input_data: InputData
 723     """
 724
 725     logging.info("  Generating the table {0} ...".
 726                  format(table.get("title", "")))
 727
 728     # Transform the data
 729     data = input_data.filter_data(table, continue_on_error=True)
 730
 731     # Prepare the header of the tables
 732     header = ["Test Case",
 733               "Trend [Mpps]",
 734               "Short-Term Change [%]",
 735               "Long-Term Change [%]",
 736               "Regressions [#]",
 737               "Progressions [#]",
 738               "Outliers [#]"
 739               ]
 740     header_str = ",".join(header) + "\n"
 741
 742     # Prepare data to the table:
 743     tbl_dict = dict()
 744     for job, builds in table["data"].items():
 745         for build in builds:
 746             for tst_name, tst_data in data[job][str(build)].iteritems():
 747                 if tst_name.lower() in table["ignore-list"]:
 748                     continue
 749                 if tbl_dict.get(tst_name, None) is None:
 750                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 751                                             "-".join(tst_data["name"].
 752                                                      split("-")[1:]))
 753                     tbl_dict[tst_name] = {"name": name,
 754                                           "data": OrderedDict()}
 755                 try:
 756                     tbl_dict[tst_name]["data"][str(build)] =  \
 757                         tst_data["result"]["throughput"]
 758                 except (TypeError, KeyError):
 759                     pass  # No data in output.xml for this test
 760
 761     tbl_lst = list()
 762     for tst_name in tbl_dict.keys():
 763         if len(tbl_dict[tst_name]["data"]) > 2:
 764
 765             pd_data = pd.Series(tbl_dict[tst_name]["data"])
 766             data_t, _ = split_outliers(pd_data, outlier_const=1.5,
 767                                        window=table["window"])
 768             last_key = data_t.keys()[-1]
 769             win_size = min(data_t.size, table["window"])
 770             win_first_idx = data_t.size - win_size
 771             key_14 = data_t.keys()[win_first_idx]
 772             long_win_size = min(data_t.size, table["long-trend-window"])
 773             median_t = data_t.rolling(window=win_size, min_periods=2).median()
 774             stdev_t = data_t.rolling(window=win_size, min_periods=2).std()
 775             median_first_idx = median_t.size - long_win_size
 776             try:
 777                 max_median = max(
 778                     [x for x in median_t.values[median_first_idx:-win_size]
 779                      if not isnan(x)])
 780             except ValueError:
 781                 max_median = nan
 782             try:
 783                 last_median_t = median_t[last_key]
 784             except KeyError:
 785                 last_median_t = nan
 786             try:
 787                 median_t_14 = median_t[key_14]
 788             except KeyError:
 789                 median_t_14 = nan
 790
 791             # Classification list:
 792             classification_lst = list()
 793             for build_nr, value in data_t.iteritems():
 794                 if isnan(median_t[build_nr]) \
 795                         or isnan(stdev_t[build_nr]) \
 796                         or isnan(value):
 797                     classification_lst.append("outlier")
 798                 elif value < (median_t[build_nr] - 3 * stdev_t[build_nr]):
 799                     classification_lst.append("regression")
 800                 elif value > (median_t[build_nr] + 3 * stdev_t[build_nr]):
 801                     classification_lst.append("progression")
 802                 else:
 803                     classification_lst.append("normal")
 804
 805             if isnan(last_median_t) or isnan(median_t_14) or median_t_14 == 0.0:
 806                 rel_change_last = nan
 807             else:
 808                 rel_change_last = round(
 809                     ((last_median_t - median_t_14) / median_t_14) * 100, 2)
 810
 811             if isnan(max_median) or isnan(last_median_t) or max_median == 0.0:
 812                 rel_change_long = nan
 813             else:
 814                 rel_change_long = round(
 815                     ((last_median_t - max_median) / max_median) * 100, 2)
 816
 817             tbl_lst.append(
 818                 [tbl_dict[tst_name]["name"],
 819                  '-' if isnan(last_median_t) else
 820                  round(last_median_t / 1000000, 2),
 821                  '-' if isnan(rel_change_last) else rel_change_last,
 822                  '-' if isnan(rel_change_long) else rel_change_long,
 823                  classification_lst[win_first_idx:].count("regression"),
 824                  classification_lst[win_first_idx:].count("progression"),
 825                  classification_lst[win_first_idx:].count("outlier")])
 826
 827     tbl_lst.sort(key=lambda rel: rel[0])
 828
 829     tbl_sorted = list()
 830     for nrr in range(table["window"], -1, -1):
 831         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
 832         for nrp in range(table["window"], -1, -1):
 833             tbl_pro = [item for item in tbl_reg if item[5] == nrp]
 834             for nro in range(table["window"], -1, -1):
 835                 tbl_out = [item for item in tbl_pro if item[6] == nro]
 836                 tbl_out.sort(key=lambda rel: rel[2])
 837                 tbl_sorted.extend(tbl_out)
 838
 839     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
 840
 841     logging.info("      Writing file: '{0}'".format(file_name))
 842     with open(file_name, "w") as file_handler:
 843         file_handler.write(header_str)
 844         for test in tbl_sorted:
 845             file_handler.write(",".join([str(item) for item in test]) + '\n')
 846
 847     txt_file_name = "{0}.txt".format(table["output-file"])
 848     txt_table = None
 849     logging.info("      Writing file: '{0}'".format(txt_file_name))
 850     with open(file_name, 'rb') as csv_file:
 851         csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 852         for row in csv_content:
 853             if txt_table is None:
 854                 txt_table = prettytable.PrettyTable(row)
 855             else:
 856                 txt_table.add_row(row)
 857         txt_table.align["Test case"] = "l"
 858     with open(txt_file_name, "w") as txt_file:
 859         txt_file.write(str(txt_table))
 860
 861
 862 def table_performance_trending_dashboard_html(table, input_data):
 863     """Generate the table(s) with algorithm:
 864     table_performance_trending_dashboard_html specified in the specification
 865     file.
 866
 867     :param table: Table to generate.
 868     :param input_data: Data to process.
 869     :type table: pandas.Series
 870     :type input_data: InputData
 871     """
 872
 873     logging.info("  Generating the table {0} ...".
 874                  format(table.get("title", "")))
 875
 876     try:
 877         with open(table["input-file"], 'rb') as csv_file:
 878             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 879             csv_lst = [item for item in csv_content]
 880     except KeyError:
 881         logging.warning("The input file is not defined.")
 882         return
 883     except csv.Error as err:
 884         logging.warning("Not possible to process the file '{0}'.\n{1}".
 885                         format(table["input-file"], err))
 886         return
 887
 888     # Table:
 889     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
 890
 891     # Table header:
 892     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
 893     for idx, item in enumerate(csv_lst[0]):
 894         alignment = "left" if idx == 0 else "center"
 895         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
 896         th.text = item
 897
 898     # Rows:
 899     colors = {"regression": ("#ffcccc", "#ff9999"),
 900               "progression": ("#c6ecc6", "#9fdf9f"),
 901               "outlier": ("#e6e6e6", "#cccccc"),
 902               "normal": ("#e9f1fb", "#d4e4f7")}
 903     for r_idx, row in enumerate(csv_lst[1:]):
 904         if int(row[4]):
 905             color = "regression"
 906         elif int(row[5]):
 907             color = "progression"
 908         elif int(row[6]):
 909             color = "outlier"
 910         else:
 911             color = "normal"
 912         background = colors[color][r_idx % 2]
 913         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
 914
 915         # Columns:
 916         for c_idx, item in enumerate(row):
 917             alignment = "left" if c_idx == 0 else "center"
 918             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
 919             # Name:
 920             url = "../trending/"
 921             file_name = ""
 922             anchor = "#"
 923             feature = ""
 924             if c_idx == 0:
 925                 if "memif" in item:
 926                     file_name = "container_memif.html"
 927
 928                 elif "vhost" in item:
 929                     if "l2xcbase" in item or "l2bdbasemaclrn" in item:
 930                         file_name = "vm_vhost_l2.html"
 931                     elif "ip4base" in item:
 932                         file_name = "vm_vhost_ip4.html"
 933
 934                 elif "ipsec" in item:
 935                     file_name = "ipsec.html"
 936
 937                 elif "ethip4lispip" in item or "ethip4vxlan" in item:
 938                     file_name = "ip4_tunnels.html"
 939
 940                 elif "ip4base" in item or "ip4scale" in item:
 941                     file_name = "ip4.html"
 942                     if "iacl" in item or "snat" in item or "cop" in item:
 943                         feature = "-features"
 944
 945                 elif "ip6base" in item or "ip6scale" in item:
 946                     file_name = "ip6.html"
 947
 948                 elif "l2xcbase" in item or "l2xcscale" in item \
 949                         or "l2bdbasemaclrn" in item or "l2bdscale" in item \
 950                         or "l2dbbasemaclrn" in item or "l2dbscale" in item:
 951                     file_name = "l2.html"
 952                     if "iacl" in item:
 953                         feature = "-features"
 954
 955                 if "x520" in item:
 956                     anchor += "x520-"
 957                 elif "x710" in item:
 958                     anchor += "x710-"
 959                 elif "xl710" in item:
 960                     anchor += "xl710-"
 961
 962                 if "64b" in item:
 963                     anchor += "64b-"
 964                 elif "78b" in item:
 965                     anchor += "78b"
 966                 elif "imix" in item:
 967                     anchor += "imix-"
 968                 elif "9000b" in item:
 969                     anchor += "9000b-"
 970                 elif "1518" in item:
 971                     anchor += "1518b-"
 972
 973                 if "1t1c" in item:
 974                     anchor += "1t1c"
 975                 elif "2t2c" in item:
 976                     anchor += "2t2c"
 977                 elif "4t4c" in item:
 978                     anchor += "4t4c"
 979
 980                 url = url + file_name + anchor + feature
 981
 982                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
 983                 ref.text = item
 984
 985             if c_idx > 0:
 986                 td.text = item
 987
 988     try:
 989         with open(table["output-file"], 'w') as html_file:
 990             logging.info("      Writing file: '{0}'".
 991                          format(table["output-file"]))
 992             html_file.write(".. raw:: html\n\n\t")
 993             html_file.write(ET.tostring(dashboard))
 994             html_file.write("\n\t<p><br><br></p>\n")
 995     except KeyError:
 996         logging.warning("The output file is not defined.")
 997         return