resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import prettytable
  21 import pandas as pd
  22
  23 from string import replace
  24 from math import isnan
  25 from xml.etree import ElementTree as ET
  26
  27 from errors import PresentationError
  28 from utils import mean, stdev, relative_change, remove_outliers, split_outliers
  29
  30
  31 def generate_tables(spec, data):
  32     """Generate all tables specified in the specification file.
  33
  34     :param spec: Specification read from the specification file.
  35     :param data: Data to process.
  36     :type spec: Specification
  37     :type data: InputData
  38     """
  39
  40     logging.info("Generating the tables ...")
  41     for table in spec.tables:
  42         try:
  43             eval(table["algorithm"])(table, data)
  44         except NameError:
  45             logging.error("The algorithm '{0}' is not defined.".
  46                           format(table["algorithm"]))
  47     logging.info("Done.")
  48
  49
  50 def table_details(table, input_data):
  51     """Generate the table(s) with algorithm: table_detailed_test_results
  52     specified in the specification file.
  53
  54     :param table: Table to generate.
  55     :param input_data: Data to process.
  56     :type table: pandas.Series
  57     :type input_data: InputData
  58     """
  59
  60     logging.info("  Generating the table {0} ...".
  61                  format(table.get("title", "")))
  62
  63     # Transform the data
  64     data = input_data.filter_data(table)
  65
  66     # Prepare the header of the tables
  67     header = list()
  68     for column in table["columns"]:
  69         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  70
  71     # Generate the data for the table according to the model in the table
  72     # specification
  73     job = table["data"].keys()[0]
  74     build = str(table["data"][job][0])
  75     try:
  76         suites = input_data.suites(job, build)
  77     except KeyError:
  78         logging.error("    No data available. The table will not be generated.")
  79         return
  80
  81     for suite_longname, suite in suites.iteritems():
  82         # Generate data
  83         suite_name = suite["name"]
  84         table_lst = list()
  85         for test in data[job][build].keys():
  86             if data[job][build][test]["parent"] in suite_name:
  87                 row_lst = list()
  88                 for column in table["columns"]:
  89                     try:
  90                         col_data = str(data[job][build][test][column["data"].
  91                                        split(" ")[1]]).replace('"', '""')
  92                         if column["data"].split(" ")[1] in ("vat-history",
  93                                                             "show-run"):
  94                             col_data = replace(col_data, " |br| ", "",
  95                                                maxreplace=1)
  96                             col_data = " |prein| {0} |preout| ".\
  97                                 format(col_data[:-5])
  98                         row_lst.append('"{0}"'.format(col_data))
  99                     except KeyError:
 100                         row_lst.append("No data")
 101                 table_lst.append(row_lst)
 102
 103         # Write the data to file
 104         if table_lst:
 105             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 106                                             table["output-file-ext"])
 107             logging.info("      Writing file: '{}'".format(file_name))
 108             with open(file_name, "w") as file_handler:
 109                 file_handler.write(",".join(header) + "\n")
 110                 for item in table_lst:
 111                     file_handler.write(",".join(item) + "\n")
 112
 113     logging.info("  Done.")
 114
 115
 116 def table_merged_details(table, input_data):
 117     """Generate the table(s) with algorithm: table_merged_details
 118     specified in the specification file.
 119
 120     :param table: Table to generate.
 121     :param input_data: Data to process.
 122     :type table: pandas.Series
 123     :type input_data: InputData
 124     """
 125
 126     logging.info("  Generating the table {0} ...".
 127                  format(table.get("title", "")))
 128
 129     # Transform the data
 130     data = input_data.filter_data(table)
 131     data = input_data.merge_data(data)
 132     data.sort_index(inplace=True)
 133
 134     suites = input_data.filter_data(table, data_set="suites")
 135     suites = input_data.merge_data(suites)
 136
 137     # Prepare the header of the tables
 138     header = list()
 139     for column in table["columns"]:
 140         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 141
 142     for _, suite in suites.iteritems():
 143         # Generate data
 144         suite_name = suite["name"]
 145         table_lst = list()
 146         for test in data.keys():
 147             if data[test]["parent"] in suite_name:
 148                 row_lst = list()
 149                 for column in table["columns"]:
 150                     try:
 151                         col_data = str(data[test][column["data"].
 152                                        split(" ")[1]]).replace('"', '""')
 153                         if column["data"].split(" ")[1] in ("vat-history",
 154                                                             "show-run"):
 155                             col_data = replace(col_data, " |br| ", "",
 156                                                maxreplace=1)
 157                             col_data = " |prein| {0} |preout| ".\
 158                                 format(col_data[:-5])
 159                         row_lst.append('"{0}"'.format(col_data))
 160                     except KeyError:
 161                         row_lst.append("No data")
 162                 table_lst.append(row_lst)
 163
 164         # Write the data to file
 165         if table_lst:
 166             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 167                                             table["output-file-ext"])
 168             logging.info("      Writing file: '{}'".format(file_name))
 169             with open(file_name, "w") as file_handler:
 170                 file_handler.write(",".join(header) + "\n")
 171                 for item in table_lst:
 172                     file_handler.write(",".join(item) + "\n")
 173
 174     logging.info("  Done.")
 175
 176
 177 def table_performance_improvements(table, input_data):
 178     """Generate the table(s) with algorithm: table_performance_improvements
 179     specified in the specification file.
 180
 181     :param table: Table to generate.
 182     :param input_data: Data to process.
 183     :type table: pandas.Series
 184     :type input_data: InputData
 185     """
 186
 187     def _write_line_to_file(file_handler, data):
 188         """Write a line to the .csv file.
 189
 190         :param file_handler: File handler for the csv file. It must be open for
 191          writing text.
 192         :param data: Item to be written to the file.
 193         :type file_handler: BinaryIO
 194         :type data: list
 195         """
 196
 197         line_lst = list()
 198         for item in data:
 199             if isinstance(item["data"], str):
 200                 # Remove -?drdisc from the end
 201                 if item["data"].endswith("drdisc"):
 202                     item["data"] = item["data"][:-8]
 203                 line_lst.append(item["data"])
 204             elif isinstance(item["data"], float):
 205                 line_lst.append("{:.1f}".format(item["data"]))
 206             elif item["data"] is None:
 207                 line_lst.append("")
 208         file_handler.write(",".join(line_lst) + "\n")
 209
 210     logging.info("  Generating the table {0} ...".
 211                  format(table.get("title", "")))
 212
 213     # Read the template
 214     file_name = table.get("template", None)
 215     if file_name:
 216         try:
 217             tmpl = _read_csv_template(file_name)
 218         except PresentationError:
 219             logging.error("  The template '{0}' does not exist. Skipping the "
 220                           "table.".format(file_name))
 221             return None
 222     else:
 223         logging.error("The template is not defined. Skipping the table.")
 224         return None
 225
 226     # Transform the data
 227     data = input_data.filter_data(table)
 228
 229     # Prepare the header of the tables
 230     header = list()
 231     for column in table["columns"]:
 232         header.append(column["title"])
 233
 234     # Generate the data for the table according to the model in the table
 235     # specification
 236     tbl_lst = list()
 237     for tmpl_item in tmpl:
 238         tbl_item = list()
 239         for column in table["columns"]:
 240             cmd = column["data"].split(" ")[0]
 241             args = column["data"].split(" ")[1:]
 242             if cmd == "template":
 243                 try:
 244                     val = float(tmpl_item[int(args[0])])
 245                 except ValueError:
 246                     val = tmpl_item[int(args[0])]
 247                 tbl_item.append({"data": val})
 248             elif cmd == "data":
 249                 jobs = args[0:-1]
 250                 operation = args[-1]
 251                 data_lst = list()
 252                 for job in jobs:
 253                     for build in data[job]:
 254                         try:
 255                             data_lst.append(float(build[tmpl_item[0]]
 256                                                   ["throughput"]["value"]))
 257                         except (KeyError, TypeError):
 258                             # No data, ignore
 259                             continue
 260                 if data_lst:
 261                     tbl_item.append({"data": (eval(operation)(data_lst)) /
 262                                              1000000})
 263                 else:
 264                     tbl_item.append({"data": None})
 265             elif cmd == "operation":
 266                 operation = args[0]
 267                 try:
 268                     nr1 = float(tbl_item[int(args[1])]["data"])
 269                     nr2 = float(tbl_item[int(args[2])]["data"])
 270                     if nr1 and nr2:
 271                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
 272                     else:
 273                         tbl_item.append({"data": None})
 274                 except (IndexError, ValueError, TypeError):
 275                     logging.error("No data for {0}".format(tbl_item[0]["data"]))
 276                     tbl_item.append({"data": None})
 277                     continue
 278             else:
 279                 logging.error("Not supported command {0}. Skipping the table.".
 280                               format(cmd))
 281                 return None
 282         tbl_lst.append(tbl_item)
 283
 284     # Sort the table according to the relative change
 285     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
 286
 287     # Create the tables and write them to the files
 288     file_names = [
 289         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
 290         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
 291         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
 292         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
 293     ]
 294
 295     for file_name in file_names:
 296         logging.info("    Writing the file '{0}'".format(file_name))
 297         with open(file_name, "w") as file_handler:
 298             file_handler.write(",".join(header) + "\n")
 299             for item in tbl_lst:
 300                 if isinstance(item[-1]["data"], float):
 301                     rel_change = round(item[-1]["data"], 1)
 302                 else:
 303                     rel_change = item[-1]["data"]
 304                 if "ndr_top" in file_name \
 305                         and "ndr" in item[0]["data"] \
 306                         and rel_change >= 10.0:
 307                     _write_line_to_file(file_handler, item)
 308                 elif "pdr_top" in file_name \
 309                         and "pdr" in item[0]["data"] \
 310                         and rel_change >= 10.0:
 311                     _write_line_to_file(file_handler, item)
 312                 elif "ndr_low" in file_name \
 313                         and "ndr" in item[0]["data"] \
 314                         and rel_change < 10.0:
 315                     _write_line_to_file(file_handler, item)
 316                 elif "pdr_low" in file_name \
 317                         and "pdr" in item[0]["data"] \
 318                         and rel_change < 10.0:
 319                     _write_line_to_file(file_handler, item)
 320
 321     logging.info("  Done.")
 322
 323
 324 def _read_csv_template(file_name):
 325     """Read the template from a .csv file.
 326
 327     :param file_name: Name / full path / relative path of the file to read.
 328     :type file_name: str
 329     :returns: Data from the template as list (lines) of lists (items on line).
 330     :rtype: list
 331     :raises: PresentationError if it is not possible to read the file.
 332     """
 333
 334     try:
 335         with open(file_name, 'r') as csv_file:
 336             tmpl_data = list()
 337             for line in csv_file:
 338                 tmpl_data.append(line[:-1].split(","))
 339         return tmpl_data
 340     except IOError as err:
 341         raise PresentationError(str(err), level="ERROR")
 342
 343
 344 def table_performance_comparison(table, input_data):
 345     """Generate the table(s) with algorithm: table_performance_comparison
 346     specified in the specification file.
 347
 348     :param table: Table to generate.
 349     :param input_data: Data to process.
 350     :type table: pandas.Series
 351     :type input_data: InputData
 352     """
 353
 354     logging.info("  Generating the table {0} ...".
 355                  format(table.get("title", "")))
 356
 357     # Transform the data
 358     data = input_data.filter_data(table, continue_on_error=True)
 359
 360     # Prepare the header of the tables
 361     try:
 362         header = ["Test case",
 363                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 364                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 365                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 366                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 367                   "Change [%]"]
 368         header_str = ",".join(header) + "\n"
 369     except (AttributeError, KeyError) as err:
 370         logging.error("The model is invalid, missing parameter: {0}".
 371                       format(err))
 372         return
 373
 374     # Prepare data to the table:
 375     tbl_dict = dict()
 376     for job, builds in table["reference"]["data"].items():
 377         for build in builds:
 378             for tst_name, tst_data in data[job][str(build)].iteritems():
 379                 if tbl_dict.get(tst_name, None) is None:
 380                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 381                                             "-".join(tst_data["name"].
 382                                                      split("-")[1:]))
 383                     tbl_dict[tst_name] = {"name": name,
 384                                           "ref-data": list(),
 385                                           "cmp-data": list()}
 386                 try:
 387                     tbl_dict[tst_name]["ref-data"].\
 388                         append(tst_data["throughput"]["value"])
 389                 except TypeError:
 390                     pass  # No data in output.xml for this test
 391
 392     for job, builds in table["compare"]["data"].items():
 393         for build in builds:
 394             for tst_name, tst_data in data[job][str(build)].iteritems():
 395                 try:
 396                     tbl_dict[tst_name]["cmp-data"].\
 397                         append(tst_data["throughput"]["value"])
 398                 except KeyError:
 399                     pass
 400                 except TypeError:
 401                     tbl_dict.pop(tst_name, None)
 402
 403     tbl_lst = list()
 404     for tst_name in tbl_dict.keys():
 405         item = [tbl_dict[tst_name]["name"], ]
 406         if tbl_dict[tst_name]["ref-data"]:
 407             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
 408                                      outlier_const=table["outlier-const"])
 409             # TODO: Specify window size.
 410             if data_t:
 411                 item.append(round(mean(data_t) / 1000000, 2))
 412                 item.append(round(stdev(data_t) / 1000000, 2))
 413             else:
 414                 item.extend([None, None])
 415         else:
 416             item.extend([None, None])
 417         if tbl_dict[tst_name]["cmp-data"]:
 418             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
 419                                      outlier_const=table["outlier-const"])
 420             # TODO: Specify window size.
 421             if data_t:
 422                 item.append(round(mean(data_t) / 1000000, 2))
 423                 item.append(round(stdev(data_t) / 1000000, 2))
 424             else:
 425                 item.extend([None, None])
 426         else:
 427             item.extend([None, None])
 428         if item[1] is not None and item[3] is not None:
 429             item.append(int(relative_change(float(item[1]), float(item[3]))))
 430         if len(item) == 6:
 431             tbl_lst.append(item)
 432
 433     # Sort the table according to the relative change
 434     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 435
 436     # Generate tables:
 437     # All tests in csv:
 438     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
 439                                                table["output-file-ext"]),
 440                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
 441                                                table["output-file-ext"]),
 442                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
 443                                                table["output-file-ext"]),
 444                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 445                                                table["output-file-ext"]),
 446                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
 447                                                table["output-file-ext"]),
 448                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
 449                                                table["output-file-ext"])
 450                  ]
 451     for file_name in tbl_names:
 452         logging.info("      Writing file: '{0}'".format(file_name))
 453         with open(file_name, "w") as file_handler:
 454             file_handler.write(header_str)
 455             for test in tbl_lst:
 456                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
 457                         file_name.split("-")[-2] in test[0]):  # cores
 458                     test[0] = "-".join(test[0].split("-")[:-1])
 459                     file_handler.write(",".join([str(item) for item in test]) +
 460                                        "\n")
 461
 462     # All tests in txt:
 463     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
 464                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
 465                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
 466                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
 467                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
 468                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
 469                      ]
 470
 471     for i, txt_name in enumerate(tbl_names_txt):
 472         txt_table = None
 473         logging.info("      Writing file: '{0}'".format(txt_name))
 474         with open(tbl_names[i], 'rb') as csv_file:
 475             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 476             for row in csv_content:
 477                 if txt_table is None:
 478                     txt_table = prettytable.PrettyTable(row)
 479                 else:
 480                     txt_table.add_row(row)
 481             txt_table.align["Test case"] = "l"
 482         with open(txt_name, "w") as txt_file:
 483             txt_file.write(str(txt_table))
 484
 485     # Selected tests in csv:
 486     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
 487                                                table["output-file-ext"])
 488     with open(input_file, "r") as in_file:
 489         lines = list()
 490         for line in in_file:
 491             lines.append(line)
 492
 493     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
 494                                                table["output-file-ext"])
 495     logging.info("      Writing file: '{0}'".format(output_file))
 496     with open(output_file, "w") as out_file:
 497         out_file.write(header_str)
 498         for i, line in enumerate(lines[1:]):
 499             if i == table["nr-of-tests-shown"]:
 500                 break
 501             out_file.write(line)
 502
 503     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
 504                                                   table["output-file-ext"])
 505     logging.info("      Writing file: '{0}'".format(output_file))
 506     with open(output_file, "w") as out_file:
 507         out_file.write(header_str)
 508         for i, line in enumerate(lines[-1:0:-1]):
 509             if i == table["nr-of-tests-shown"]:
 510                 break
 511             out_file.write(line)
 512
 513     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 514                                                table["output-file-ext"])
 515     with open(input_file, "r") as in_file:
 516         lines = list()
 517         for line in in_file:
 518             lines.append(line)
 519
 520     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
 521                                                table["output-file-ext"])
 522     logging.info("      Writing file: '{0}'".format(output_file))
 523     with open(output_file, "w") as out_file:
 524         out_file.write(header_str)
 525         for i, line in enumerate(lines[1:]):
 526             if i == table["nr-of-tests-shown"]:
 527                 break
 528             out_file.write(line)
 529
 530     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
 531                                                   table["output-file-ext"])
 532     logging.info("      Writing file: '{0}'".format(output_file))
 533     with open(output_file, "w") as out_file:
 534         out_file.write(header_str)
 535         for i, line in enumerate(lines[-1:0:-1]):
 536             if i == table["nr-of-tests-shown"]:
 537                 break
 538             out_file.write(line)
 539
 540
 541 def table_performance_comparison_mrr(table, input_data):
 542     """Generate the table(s) with algorithm: table_performance_comparison_mrr
 543     specified in the specification file.
 544
 545     :param table: Table to generate.
 546     :param input_data: Data to process.
 547     :type table: pandas.Series
 548     :type input_data: InputData
 549     """
 550
 551     logging.info("  Generating the table {0} ...".
 552                  format(table.get("title", "")))
 553
 554     # Transform the data
 555     data = input_data.filter_data(table, continue_on_error=True)
 556
 557     # Prepare the header of the tables
 558     try:
 559         header = ["Test case",
 560                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 561                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 562                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 563                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 564                   "Change [%]"]
 565         header_str = ",".join(header) + "\n"
 566     except (AttributeError, KeyError) as err:
 567         logging.error("The model is invalid, missing parameter: {0}".
 568                       format(err))
 569         return
 570
 571     # Prepare data to the table:
 572     tbl_dict = dict()
 573     for job, builds in table["reference"]["data"].items():
 574         for build in builds:
 575             for tst_name, tst_data in data[job][str(build)].iteritems():
 576                 if tbl_dict.get(tst_name, None) is None:
 577                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 578                                             "-".join(tst_data["name"].
 579                                                      split("-")[1:]))
 580                     tbl_dict[tst_name] = {"name": name,
 581                                           "ref-data": list(),
 582                                           "cmp-data": list()}
 583                 try:
 584                     tbl_dict[tst_name]["ref-data"].\
 585                         append(tst_data["result"]["throughput"])
 586                 except TypeError:
 587                     pass  # No data in output.xml for this test
 588
 589     for job, builds in table["compare"]["data"].items():
 590         for build in builds:
 591             for tst_name, tst_data in data[job][str(build)].iteritems():
 592                 try:
 593                     tbl_dict[tst_name]["cmp-data"].\
 594                         append(tst_data["result"]["throughput"])
 595                 except KeyError:
 596                     pass
 597                 except TypeError:
 598                     tbl_dict.pop(tst_name, None)
 599
 600     tbl_lst = list()
 601     for tst_name in tbl_dict.keys():
 602         item = [tbl_dict[tst_name]["name"], ]
 603         if tbl_dict[tst_name]["ref-data"]:
 604             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
 605                                      outlier_const=table["outlier-const"])
 606             # TODO: Specify window size.
 607             if data_t:
 608                 item.append(round(mean(data_t) / 1000000, 2))
 609                 item.append(round(stdev(data_t) / 1000000, 2))
 610             else:
 611                 item.extend([None, None])
 612         else:
 613             item.extend([None, None])
 614         if tbl_dict[tst_name]["cmp-data"]:
 615             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
 616                                      outlier_const=table["outlier-const"])
 617             # TODO: Specify window size.
 618             if data_t:
 619                 item.append(round(mean(data_t) / 1000000, 2))
 620                 item.append(round(stdev(data_t) / 1000000, 2))
 621             else:
 622                 item.extend([None, None])
 623         else:
 624             item.extend([None, None])
 625         if item[1] is not None and item[3] is not None and item[1] != 0:
 626             item.append(int(relative_change(float(item[1]), float(item[3]))))
 627         if len(item) == 6:
 628             tbl_lst.append(item)
 629
 630     # Sort the table according to the relative change
 631     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 632
 633     # Generate tables:
 634     # All tests in csv:
 635     tbl_names = ["{0}-1t1c-full{1}".format(table["output-file"],
 636                                            table["output-file-ext"]),
 637                  "{0}-2t2c-full{1}".format(table["output-file"],
 638                                            table["output-file-ext"]),
 639                  "{0}-4t4c-full{1}".format(table["output-file"],
 640                                            table["output-file-ext"])
 641                  ]
 642     for file_name in tbl_names:
 643         logging.info("      Writing file: '{0}'".format(file_name))
 644         with open(file_name, "w") as file_handler:
 645             file_handler.write(header_str)
 646             for test in tbl_lst:
 647                 if file_name.split("-")[-2] in test[0]:  # cores
 648                     test[0] = "-".join(test[0].split("-")[:-1])
 649                     file_handler.write(",".join([str(item) for item in test]) +
 650                                        "\n")
 651
 652     # All tests in txt:
 653     tbl_names_txt = ["{0}-1t1c-full.txt".format(table["output-file"]),
 654                      "{0}-2t2c-full.txt".format(table["output-file"]),
 655                      "{0}-4t4c-full.txt".format(table["output-file"])
 656                      ]
 657
 658     for i, txt_name in enumerate(tbl_names_txt):
 659         txt_table = None
 660         logging.info("      Writing file: '{0}'".format(txt_name))
 661         with open(tbl_names[i], 'rb') as csv_file:
 662             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 663             for row in csv_content:
 664                 if txt_table is None:
 665                     txt_table = prettytable.PrettyTable(row)
 666                 else:
 667                     txt_table.add_row(row)
 668             txt_table.align["Test case"] = "l"
 669         with open(txt_name, "w") as txt_file:
 670             txt_file.write(str(txt_table))
 671
 672
 673 def table_performance_trending_dashboard(table, input_data):
 674     """Generate the table(s) with algorithm: table_performance_comparison
 675     specified in the specification file.
 676
 677     :param table: Table to generate.
 678     :param input_data: Data to process.
 679     :type table: pandas.Series
 680     :type input_data: InputData
 681     """
 682
 683     logging.info("  Generating the table {0} ...".
 684                  format(table.get("title", "")))
 685
 686     # Transform the data
 687     data = input_data.filter_data(table, continue_on_error=True)
 688
 689     # Prepare the header of the tables
 690     header = ["Test Case",
 691               "Throughput Trend [Mpps]",
 692               "Long Trend Compliance",
 693               "Trend Compliance",
 694               "Top Anomaly [Mpps]",
 695               "Change [%]",
 696               "Outliers [Number]"
 697               ]
 698     header_str = ",".join(header) + "\n"
 699
 700     # Prepare data to the table:
 701     tbl_dict = dict()
 702     for job, builds in table["data"].items():
 703         for build in builds:
 704             for tst_name, tst_data in data[job][str(build)].iteritems():
 705                 if tbl_dict.get(tst_name, None) is None:
 706                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 707                                             "-".join(tst_data["name"].
 708                                                      split("-")[1:]))
 709                     tbl_dict[tst_name] = {"name": name,
 710                                           "data": dict()}
 711                 try:
 712                     tbl_dict[tst_name]["data"][str(build)] =  \
 713                         tst_data["result"]["throughput"]
 714                 except (TypeError, KeyError):
 715                     pass  # No data in output.xml for this test
 716
 717     tbl_lst = list()
 718     for tst_name in tbl_dict.keys():
 719         if len(tbl_dict[tst_name]["data"]) > 2:
 720
 721             pd_data = pd.Series(tbl_dict[tst_name]["data"])
 722             win_size = min(pd_data.size, table["window"])
 723             # Test name:
 724             name = tbl_dict[tst_name]["name"]
 725
 726             median = pd_data.rolling(window=win_size, min_periods=2).median()
 727             median_idx = pd_data.size - table["long-trend-window"]
 728             median_idx = 0 if median_idx < 0 else median_idx
 729             max_median = max(median.values[median_idx:])
 730             trimmed_data, _ = split_outliers(pd_data, outlier_const=1.5,
 731                                              window=win_size)
 732             stdev_t = pd_data.rolling(window=win_size, min_periods=2).std()
 733
 734             rel_change_lst = [None, ]
 735             classification_lst = [None, ]
 736             median_lst = [None, ]
 737             sample_lst = [None, ]
 738             first = True
 739             for build_nr, value in pd_data.iteritems():
 740                 if first:
 741                     first = False
 742                     continue
 743                 # Relative changes list:
 744                 if not isnan(value) \
 745                         and not isnan(median[build_nr]) \
 746                         and median[build_nr] != 0:
 747                     rel_change_lst.append(round(
 748                         relative_change(float(median[build_nr]), float(value)),
 749                         2))
 750                 else:
 751                     rel_change_lst.append(None)
 752
 753                 # Classification list:
 754                 if isnan(trimmed_data[build_nr]) \
 755                         or isnan(median[build_nr]) \
 756                         or isnan(stdev_t[build_nr]) \
 757                         or isnan(value):
 758                     classification_lst.append("outlier")
 759                 elif value < (median[build_nr] - 3 * stdev_t[build_nr]):
 760                     classification_lst.append("regression")
 761                 elif value > (median[build_nr] + 3 * stdev_t[build_nr]):
 762                     classification_lst.append("progression")
 763                 else:
 764                     classification_lst.append("normal")
 765                 sample_lst.append(value)
 766                 median_lst.append(median[build_nr])
 767
 768             last_idx = len(classification_lst) - 1
 769             first_idx = last_idx - int(table["evaluated-window"])
 770             if first_idx < 0:
 771                 first_idx = 0
 772
 773             nr_outliers = 0
 774             consecutive_outliers = 0
 775             failure = False
 776             for item in classification_lst[first_idx:]:
 777                 if item == "outlier":
 778                     nr_outliers += 1
 779                     consecutive_outliers += 1
 780                     if consecutive_outliers == 3:
 781                         failure = True
 782                 else:
 783                     consecutive_outliers = 0
 784
 785             if failure:
 786                 classification = "failure"
 787             elif "regression" in classification_lst[first_idx:]:
 788                 classification = "regression"
 789             elif "progression" in classification_lst[first_idx:]:
 790                 classification = "progression"
 791             else:
 792                 classification = "normal"
 793
 794             if classification == "normal":
 795                 index = len(classification_lst) - 1
 796             else:
 797                 tmp_classification = "outlier" if classification == "failure" \
 798                     else classification
 799                 index = None
 800                 for idx in range(first_idx, len(classification_lst)):
 801                     if classification_lst[idx] == tmp_classification:
 802                         if rel_change_lst[idx]:
 803                             index = idx
 804                             break
 805                 if index is None:
 806                     continue
 807                 for idx in range(index+1, len(classification_lst)):
 808                     if classification_lst[idx] == tmp_classification:
 809                         if rel_change_lst[idx]:
 810                             if (abs(rel_change_lst[idx]) >
 811                                     abs(rel_change_lst[index])):
 812                                 index = idx
 813
 814             logging.debug("{}".format(name))
 815             logging.debug("sample_lst: {} - {}".
 816                           format(len(sample_lst), sample_lst))
 817             logging.debug("median_lst: {} - {}".
 818                           format(len(median_lst), median_lst))
 819             logging.debug("rel_change: {} - {}".
 820                           format(len(rel_change_lst), rel_change_lst))
 821             logging.debug("classn_lst: {} - {}".
 822                           format(len(classification_lst), classification_lst))
 823             logging.debug("index:      {}".format(index))
 824             logging.debug("classifica: {}".format(classification))
 825
 826             try:
 827                 trend = round(float(median_lst[-1]) / 1000000, 2) \
 828                     if not isnan(median_lst[-1]) else '-'
 829                 sample = round(float(sample_lst[index]) / 1000000, 2) \
 830                     if not isnan(sample_lst[index]) else '-'
 831                 rel_change = rel_change_lst[index] \
 832                     if rel_change_lst[index] is not None else '-'
 833                 if not isnan(max_median):
 834                     if not isnan(sample_lst[index]):
 835                         long_trend_threshold = \
 836                             max_median * (table["long-trend-threshold"] / 100)
 837                         if sample_lst[index] < long_trend_threshold:
 838                             long_trend_classification = "failure"
 839                         else:
 840                             long_trend_classification = 'normal'
 841                     else:
 842                         long_trend_classification = "failure"
 843                 else:
 844                     long_trend_classification = '-'
 845                 tbl_lst.append([name,
 846                                 trend,
 847                                 long_trend_classification,
 848                                 classification,
 849                                 '-' if classification == "normal" else sample,
 850                                 '-' if classification == "normal" else
 851                                 rel_change,
 852                                 nr_outliers])
 853             except IndexError as err:
 854                 logging.error("{}".format(err))
 855                 continue
 856
 857     # Sort the table according to the classification
 858     tbl_sorted = list()
 859     for long_trend_class in ("failure", 'normal', '-'):
 860         tbl_long = [item for item in tbl_lst if item[2] == long_trend_class]
 861         for classification in \
 862                 ("failure", "regression", "progression", "normal"):
 863             tbl_tmp = [item for item in tbl_long if item[3] == classification]
 864             tbl_tmp.sort(key=lambda rel: rel[0])
 865             tbl_sorted.extend(tbl_tmp)
 866
 867     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
 868
 869     logging.info("      Writing file: '{0}'".format(file_name))
 870     with open(file_name, "w") as file_handler:
 871         file_handler.write(header_str)
 872         for test in tbl_sorted:
 873             file_handler.write(",".join([str(item) for item in test]) + '\n')
 874
 875     txt_file_name = "{0}.txt".format(table["output-file"])
 876     txt_table = None
 877     logging.info("      Writing file: '{0}'".format(txt_file_name))
 878     with open(file_name, 'rb') as csv_file:
 879         csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 880         for row in csv_content:
 881             if txt_table is None:
 882                 txt_table = prettytable.PrettyTable(row)
 883             else:
 884                 txt_table.add_row(row)
 885         txt_table.align["Test case"] = "l"
 886     with open(txt_file_name, "w") as txt_file:
 887         txt_file.write(str(txt_table))
 888
 889
 890 def table_performance_trending_dashboard_html(table, input_data):
 891     """Generate the table(s) with algorithm:
 892     table_performance_trending_dashboard_html specified in the specification
 893     file.
 894
 895     :param table: Table to generate.
 896     :param input_data: Data to process.
 897     :type table: pandas.Series
 898     :type input_data: InputData
 899     """
 900
 901     logging.info("  Generating the table {0} ...".
 902                  format(table.get("title", "")))
 903
 904     try:
 905         with open(table["input-file"], 'rb') as csv_file:
 906             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 907             csv_lst = [item for item in csv_content]
 908     except KeyError:
 909         logging.warning("The input file is not defined.")
 910         return
 911     except csv.Error as err:
 912         logging.warning("Not possible to process the file '{0}'.\n{1}".
 913                         format(table["input-file"], err))
 914         return
 915
 916     # Table:
 917     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
 918
 919     # Table header:
 920     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
 921     for idx, item in enumerate(csv_lst[0]):
 922         alignment = "left" if idx == 0 else "center"
 923         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
 924         th.text = item
 925
 926     # Rows:
 927     for r_idx, row in enumerate(csv_lst[1:]):
 928         background = "#D4E4F7" if r_idx % 2 else "white"
 929         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
 930
 931         # Columns:
 932         for c_idx, item in enumerate(row):
 933             alignment = "left" if c_idx == 0 else "center"
 934             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
 935             # Name:
 936             url = "../trending/"
 937             file_name = ""
 938             anchor = "#"
 939             feature = ""
 940             if c_idx == 0:
 941                 if "memif" in item:
 942                     file_name = "container_memif.html"
 943
 944                 elif "vhost" in item:
 945                     if "l2xcbase" in item or "l2bdbasemaclrn" in item:
 946                         file_name = "vm_vhost_l2.html"
 947                     elif "ip4base" in item:
 948                         file_name = "vm_vhost_ip4.html"
 949
 950                 elif "ipsec" in item:
 951                     file_name = "ipsec.html"
 952
 953                 elif "ethip4lispip" in item or "ethip4vxlan" in item:
 954                     file_name = "ip4_tunnels.html"
 955
 956                 elif "ip4base" in item or "ip4scale" in item:
 957                     file_name = "ip4.html"
 958                     if "iacl" in item or "snat" in item or "cop" in item:
 959                         feature = "-features"
 960
 961                 elif "ip6base" in item or "ip6scale" in item:
 962                     file_name = "ip6.html"
 963
 964                 elif "l2xcbase" in item or "l2xcscale" in item \
 965                         or "l2bdbasemaclrn" in item or "l2bdscale" in item \
 966                         or "l2dbbasemaclrn" in item or "l2dbscale" in item:
 967                     file_name = "l2.html"
 968                     if "iacl" in item:
 969                         feature = "-features"
 970
 971                 if "x520" in item:
 972                     anchor += "x520-"
 973                 elif "x710" in item:
 974                     anchor += "x710-"
 975                 elif "xl710" in item:
 976                     anchor += "xl710-"
 977
 978                 if "64b" in item:
 979                     anchor += "64b-"
 980                 elif "78b" in item:
 981                     anchor += "78b"
 982                 elif "imix" in item:
 983                     anchor += "imix-"
 984                 elif "9000b" in item:
 985                     anchor += "9000b-"
 986                 elif "1518" in item:
 987                     anchor += "1518b-"
 988
 989                 if "1t1c" in item:
 990                     anchor += "1t1c"
 991                 elif "2t2c" in item:
 992                     anchor += "2t2c"
 993                 elif "4t4c" in item:
 994                     anchor += "4t4c"
 995
 996                 url = url + file_name + anchor + feature
 997
 998                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
 999                 ref.text = item
1000
1001             if c_idx == 3:
1002                 if item == "regression":
1003                     td.set("bgcolor", "#eca1a6")
1004                 elif item == "failure":
1005                     td.set("bgcolor", "#d6cbd3")
1006                 elif item == "progression":
1007                     td.set("bgcolor", "#bdcebe")
1008             if c_idx > 0:
1009                 td.text = item
1010
1011     try:
1012         with open(table["output-file"], 'w') as html_file:
1013             logging.info("      Writing file: '{0}'".
1014                          format(table["output-file"]))
1015             html_file.write(".. raw:: html\n\n\t")
1016             html_file.write(ET.tostring(dashboard))
1017             html_file.write("\n\t<p><br><br></p>\n")
1018     except KeyError:
1019         logging.warning("The output file is not defined.")
1020         return