resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import prettytable
  21
  22 from string import replace
  23
  24 from errors import PresentationError
  25 from utils import mean, stdev, relative_change, find_outliers
  26
  27
  28 def generate_tables(spec, data):
  29     """Generate all tables specified in the specification file.
  30
  31     :param spec: Specification read from the specification file.
  32     :param data: Data to process.
  33     :type spec: Specification
  34     :type data: InputData
  35     """
  36
  37     logging.info("Generating the tables ...")
  38     for table in spec.tables:
  39         try:
  40             eval(table["algorithm"])(table, data)
  41         except NameError:
  42             logging.error("The algorithm '{0}' is not defined.".
  43                           format(table["algorithm"]))
  44     logging.info("Done.")
  45
  46
  47 def table_details(table, input_data):
  48     """Generate the table(s) with algorithm: table_detailed_test_results
  49     specified in the specification file.
  50
  51     :param table: Table to generate.
  52     :param input_data: Data to process.
  53     :type table: pandas.Series
  54     :type input_data: InputData
  55     """
  56
  57     logging.info("  Generating the table {0} ...".
  58                  format(table.get("title", "")))
  59
  60     # Transform the data
  61     data = input_data.filter_data(table)
  62
  63     # Prepare the header of the tables
  64     header = list()
  65     for column in table["columns"]:
  66         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  67
  68     # Generate the data for the table according to the model in the table
  69     # specification
  70     job = table["data"].keys()[0]
  71     build = str(table["data"][job][0])
  72     try:
  73         suites = input_data.suites(job, build)
  74     except KeyError:
  75         logging.error("    No data available. The table will not be generated.")
  76         return
  77
  78     for suite_longname, suite in suites.iteritems():
  79         # Generate data
  80         suite_name = suite["name"]
  81         table_lst = list()
  82         for test in data[job][build].keys():
  83             if data[job][build][test]["parent"] in suite_name:
  84                 row_lst = list()
  85                 for column in table["columns"]:
  86                     try:
  87                         col_data = str(data[job][build][test][column["data"].
  88                                        split(" ")[1]]).replace('"', '""')
  89                         if column["data"].split(" ")[1] in ("vat-history",
  90                                                             "show-run"):
  91                             col_data = replace(col_data, " |br| ", "",
  92                                                maxreplace=1)
  93                             col_data = " |prein| {0} |preout| ".\
  94                                 format(col_data[:-5])
  95                         row_lst.append('"{0}"'.format(col_data))
  96                     except KeyError:
  97                         row_lst.append("No data")
  98                 table_lst.append(row_lst)
  99
 100         # Write the data to file
 101         if table_lst:
 102             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 103                                             table["output-file-ext"])
 104             logging.info("      Writing file: '{}'".format(file_name))
 105             with open(file_name, "w") as file_handler:
 106                 file_handler.write(",".join(header) + "\n")
 107                 for item in table_lst:
 108                     file_handler.write(",".join(item) + "\n")
 109
 110     logging.info("  Done.")
 111
 112
 113 def table_merged_details(table, input_data):
 114     """Generate the table(s) with algorithm: table_merged_details
 115     specified in the specification file.
 116
 117     :param table: Table to generate.
 118     :param input_data: Data to process.
 119     :type table: pandas.Series
 120     :type input_data: InputData
 121     """
 122
 123     logging.info("  Generating the table {0} ...".
 124                  format(table.get("title", "")))
 125
 126     # Transform the data
 127     data = input_data.filter_data(table)
 128     data = input_data.merge_data(data)
 129     data.sort_index(inplace=True)
 130
 131     suites = input_data.filter_data(table, data_set="suites")
 132     suites = input_data.merge_data(suites)
 133
 134     # Prepare the header of the tables
 135     header = list()
 136     for column in table["columns"]:
 137         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 138
 139     for _, suite in suites.iteritems():
 140         # Generate data
 141         suite_name = suite["name"]
 142         table_lst = list()
 143         for test in data.keys():
 144             if data[test]["parent"] in suite_name:
 145                 row_lst = list()
 146                 for column in table["columns"]:
 147                     try:
 148                         col_data = str(data[test][column["data"].
 149                                        split(" ")[1]]).replace('"', '""')
 150                         if column["data"].split(" ")[1] in ("vat-history",
 151                                                             "show-run"):
 152                             col_data = replace(col_data, " |br| ", "",
 153                                                maxreplace=1)
 154                             col_data = " |prein| {0} |preout| ".\
 155                                 format(col_data[:-5])
 156                         row_lst.append('"{0}"'.format(col_data))
 157                     except KeyError:
 158                         row_lst.append("No data")
 159                 table_lst.append(row_lst)
 160
 161         # Write the data to file
 162         if table_lst:
 163             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 164                                             table["output-file-ext"])
 165             logging.info("      Writing file: '{}'".format(file_name))
 166             with open(file_name, "w") as file_handler:
 167                 file_handler.write(",".join(header) + "\n")
 168                 for item in table_lst:
 169                     file_handler.write(",".join(item) + "\n")
 170
 171     logging.info("  Done.")
 172
 173
 174 def table_performance_improvements(table, input_data):
 175     """Generate the table(s) with algorithm: table_performance_improvements
 176     specified in the specification file.
 177
 178     :param table: Table to generate.
 179     :param input_data: Data to process.
 180     :type table: pandas.Series
 181     :type input_data: InputData
 182     """
 183
 184     def _write_line_to_file(file_handler, data):
 185         """Write a line to the .csv file.
 186
 187         :param file_handler: File handler for the csv file. It must be open for
 188          writing text.
 189         :param data: Item to be written to the file.
 190         :type file_handler: BinaryIO
 191         :type data: list
 192         """
 193
 194         line_lst = list()
 195         for item in data:
 196             if isinstance(item["data"], str):
 197                 # Remove -?drdisc from the end
 198                 if item["data"].endswith("drdisc"):
 199                     item["data"] = item["data"][:-8]
 200                 line_lst.append(item["data"])
 201             elif isinstance(item["data"], float):
 202                 line_lst.append("{:.1f}".format(item["data"]))
 203             elif item["data"] is None:
 204                 line_lst.append("")
 205         file_handler.write(",".join(line_lst) + "\n")
 206
 207     logging.info("  Generating the table {0} ...".
 208                  format(table.get("title", "")))
 209
 210     # Read the template
 211     file_name = table.get("template", None)
 212     if file_name:
 213         try:
 214             tmpl = _read_csv_template(file_name)
 215         except PresentationError:
 216             logging.error("  The template '{0}' does not exist. Skipping the "
 217                           "table.".format(file_name))
 218             return None
 219     else:
 220         logging.error("The template is not defined. Skipping the table.")
 221         return None
 222
 223     # Transform the data
 224     data = input_data.filter_data(table)
 225
 226     # Prepare the header of the tables
 227     header = list()
 228     for column in table["columns"]:
 229         header.append(column["title"])
 230
 231     # Generate the data for the table according to the model in the table
 232     # specification
 233     tbl_lst = list()
 234     for tmpl_item in tmpl:
 235         tbl_item = list()
 236         for column in table["columns"]:
 237             cmd = column["data"].split(" ")[0]
 238             args = column["data"].split(" ")[1:]
 239             if cmd == "template":
 240                 try:
 241                     val = float(tmpl_item[int(args[0])])
 242                 except ValueError:
 243                     val = tmpl_item[int(args[0])]
 244                 tbl_item.append({"data": val})
 245             elif cmd == "data":
 246                 jobs = args[0:-1]
 247                 operation = args[-1]
 248                 data_lst = list()
 249                 for job in jobs:
 250                     for build in data[job]:
 251                         try:
 252                             data_lst.append(float(build[tmpl_item[0]]
 253                                                   ["throughput"]["value"]))
 254                         except (KeyError, TypeError):
 255                             # No data, ignore
 256                             continue
 257                 if data_lst:
 258                     tbl_item.append({"data": (eval(operation)(data_lst)) /
 259                                              1000000})
 260                 else:
 261                     tbl_item.append({"data": None})
 262             elif cmd == "operation":
 263                 operation = args[0]
 264                 try:
 265                     nr1 = float(tbl_item[int(args[1])]["data"])
 266                     nr2 = float(tbl_item[int(args[2])]["data"])
 267                     if nr1 and nr2:
 268                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
 269                     else:
 270                         tbl_item.append({"data": None})
 271                 except (IndexError, ValueError, TypeError):
 272                     logging.error("No data for {0}".format(tbl_item[0]["data"]))
 273                     tbl_item.append({"data": None})
 274                     continue
 275             else:
 276                 logging.error("Not supported command {0}. Skipping the table.".
 277                               format(cmd))
 278                 return None
 279         tbl_lst.append(tbl_item)
 280
 281     # Sort the table according to the relative change
 282     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
 283
 284     # Create the tables and write them to the files
 285     file_names = [
 286         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
 287         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
 288         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
 289         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
 290     ]
 291
 292     for file_name in file_names:
 293         logging.info("    Writing the file '{0}'".format(file_name))
 294         with open(file_name, "w") as file_handler:
 295             file_handler.write(",".join(header) + "\n")
 296             for item in tbl_lst:
 297                 if isinstance(item[-1]["data"], float):
 298                     rel_change = round(item[-1]["data"], 1)
 299                 else:
 300                     rel_change = item[-1]["data"]
 301                 if "ndr_top" in file_name \
 302                         and "ndr" in item[0]["data"] \
 303                         and rel_change >= 10.0:
 304                     _write_line_to_file(file_handler, item)
 305                 elif "pdr_top" in file_name \
 306                         and "pdr" in item[0]["data"] \
 307                         and rel_change >= 10.0:
 308                     _write_line_to_file(file_handler, item)
 309                 elif "ndr_low" in file_name \
 310                         and "ndr" in item[0]["data"] \
 311                         and rel_change < 10.0:
 312                     _write_line_to_file(file_handler, item)
 313                 elif "pdr_low" in file_name \
 314                         and "pdr" in item[0]["data"] \
 315                         and rel_change < 10.0:
 316                     _write_line_to_file(file_handler, item)
 317
 318     logging.info("  Done.")
 319
 320
 321 def _read_csv_template(file_name):
 322     """Read the template from a .csv file.
 323
 324     :param file_name: Name / full path / relative path of the file to read.
 325     :type file_name: str
 326     :returns: Data from the template as list (lines) of lists (items on line).
 327     :rtype: list
 328     :raises: PresentationError if it is not possible to read the file.
 329     """
 330
 331     try:
 332         with open(file_name, 'r') as csv_file:
 333             tmpl_data = list()
 334             for line in csv_file:
 335                 tmpl_data.append(line[:-1].split(","))
 336         return tmpl_data
 337     except IOError as err:
 338         raise PresentationError(str(err), level="ERROR")
 339
 340
 341 def table_performance_comparison(table, input_data):
 342     """Generate the table(s) with algorithm: table_performance_comparison
 343     specified in the specification file.
 344
 345     :param table: Table to generate.
 346     :param input_data: Data to process.
 347     :type table: pandas.Series
 348     :type input_data: InputData
 349     """
 350
 351     logging.info("  Generating the table {0} ...".
 352                  format(table.get("title", "")))
 353
 354     # Transform the data
 355     data = input_data.filter_data(table)
 356
 357     # Prepare the header of the tables
 358     try:
 359         header = ["Test case",
 360                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 361                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 362                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 363                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 364                   "Change [%]"]
 365         header_str = ",".join(header) + "\n"
 366     except (AttributeError, KeyError) as err:
 367         logging.error("The model is invalid, missing parameter: {0}".
 368                       format(err))
 369         return
 370
 371     # Prepare data to the table:
 372     tbl_dict = dict()
 373     for job, builds in table["reference"]["data"].items():
 374         for build in builds:
 375             for tst_name, tst_data in data[job][str(build)].iteritems():
 376                 if tbl_dict.get(tst_name, None) is None:
 377                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 378                                             "-".join(tst_data["name"].
 379                                                      split("-")[1:]))
 380                     tbl_dict[tst_name] = {"name": name,
 381                                           "ref-data": list(),
 382                                           "cmp-data": list()}
 383                 try:
 384                     tbl_dict[tst_name]["ref-data"].\
 385                         append(tst_data["throughput"]["value"])
 386                 except TypeError:
 387                     pass  # No data in output.xml for this test
 388
 389     for job, builds in table["compare"]["data"].items():
 390         for build in builds:
 391             for tst_name, tst_data in data[job][str(build)].iteritems():
 392                 try:
 393                     tbl_dict[tst_name]["cmp-data"].\
 394                         append(tst_data["throughput"]["value"])
 395                 except KeyError:
 396                     pass
 397                 except TypeError:
 398                     tbl_dict.pop(tst_name, None)
 399
 400     tbl_lst = list()
 401     for tst_name in tbl_dict.keys():
 402         item = [tbl_dict[tst_name]["name"], ]
 403         if tbl_dict[tst_name]["ref-data"]:
 404             data_t, _ = find_outliers(tbl_dict[tst_name]["ref-data"],
 405                                       table["outlier-const"])
 406             item.append(round(mean(data_t) / 1000000, 2))
 407             item.append(round(stdev(data_t) / 1000000, 2))
 408         else:
 409             item.extend([None, None])
 410         if tbl_dict[tst_name]["cmp-data"]:
 411             data_t, _ = find_outliers(tbl_dict[tst_name]["cmp-data"],
 412                                       table["outlier-const"])
 413             item.append(round(mean(data_t) / 1000000, 2))
 414             item.append(round(stdev(data_t) / 1000000, 2))
 415         else:
 416             item.extend([None, None])
 417         if item[1] is not None and item[3] is not None:
 418             item.append(int(relative_change(float(item[1]), float(item[3]))))
 419         if len(item) == 6:
 420             tbl_lst.append(item)
 421
 422     # Sort the table according to the relative change
 423     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 424
 425     # Generate tables:
 426     # All tests in csv:
 427     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
 428                                                table["output-file-ext"]),
 429                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
 430                                                table["output-file-ext"]),
 431                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
 432                                                table["output-file-ext"]),
 433                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 434                                                table["output-file-ext"]),
 435                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
 436                                                table["output-file-ext"]),
 437                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
 438                                                table["output-file-ext"])
 439                  ]
 440     for file_name in tbl_names:
 441         logging.info("      Writing file: '{}'".format(file_name))
 442         with open(file_name, "w") as file_handler:
 443             file_handler.write(header_str)
 444             for test in tbl_lst:
 445                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
 446                         file_name.split("-")[-2] in test[0]):  # cores
 447                     test[0] = "-".join(test[0].split("-")[:-1])
 448                     file_handler.write(",".join([str(item) for item in test]) +
 449                                        "\n")
 450
 451     # All tests in txt:
 452     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
 453                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
 454                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
 455                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
 456                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
 457                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
 458                      ]
 459
 460     for i, txt_name in enumerate(tbl_names_txt):
 461         txt_table = None
 462         logging.info("      Writing file: '{}'".format(txt_name))
 463         with open(tbl_names[i], 'rb') as csv_file:
 464             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 465             for row in csv_content:
 466                 if txt_table is None:
 467                     txt_table = prettytable.PrettyTable(row)
 468                 else:
 469                     txt_table.add_row(row)
 470             txt_table.align["Test case"] = "l"
 471         with open(txt_name, "w") as txt_file:
 472             txt_file.write(str(txt_table))
 473
 474     # Selected tests in csv:
 475     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
 476                                                table["output-file-ext"])
 477     with open(input_file, "r") as in_file:
 478         lines = list()
 479         for line in in_file:
 480             lines.append(line)
 481
 482     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
 483                                                table["output-file-ext"])
 484     logging.info("      Writing file: '{}'".format(output_file))
 485     with open(output_file, "w") as out_file:
 486         out_file.write(header_str)
 487         for i, line in enumerate(lines[1:]):
 488             if i == table["nr-of-tests-shown"]:
 489                 break
 490             out_file.write(line)
 491
 492     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
 493                                                   table["output-file-ext"])
 494     logging.info("      Writing file: '{}'".format(output_file))
 495     with open(output_file, "w") as out_file:
 496         out_file.write(header_str)
 497         for i, line in enumerate(lines[-1:0:-1]):
 498             if i == table["nr-of-tests-shown"]:
 499                 break
 500             out_file.write(line)
 501
 502     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 503                                                table["output-file-ext"])
 504     with open(input_file, "r") as in_file:
 505         lines = list()
 506         for line in in_file:
 507             lines.append(line)
 508
 509     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
 510                                                table["output-file-ext"])
 511     logging.info("      Writing file: '{}'".format(output_file))
 512     with open(output_file, "w") as out_file:
 513         out_file.write(header_str)
 514         for i, line in enumerate(lines[1:]):
 515             if i == table["nr-of-tests-shown"]:
 516                 break
 517             out_file.write(line)
 518
 519     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
 520                                                   table["output-file-ext"])
 521     logging.info("      Writing file: '{}'".format(output_file))
 522     with open(output_file, "w") as out_file:
 523         out_file.write(header_str)
 524         for i, line in enumerate(lines[-1:0:-1]):
 525             if i == table["nr-of-tests-shown"]:
 526                 break
 527             out_file.write(line)