resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import prettytable
  21
  22 from string import replace
  23
  24 from errors import PresentationError
  25 from utils import mean, stdev, relative_change
  26
  27
  28 def generate_tables(spec, data):
  29     """Generate all tables specified in the specification file.
  30
  31     :param spec: Specification read from the specification file.
  32     :param data: Data to process.
  33     :type spec: Specification
  34     :type data: InputData
  35     """
  36
  37     logging.info("Generating the tables ...")
  38     for table in spec.tables:
  39         try:
  40             eval(table["algorithm"])(table, data)
  41         except NameError:
  42             logging.error("The algorithm '{0}' is not defined.".
  43                           format(table["algorithm"]))
  44     logging.info("Done.")
  45
  46
  47 def table_details(table, input_data):
  48     """Generate the table(s) with algorithm: table_detailed_test_results
  49     specified in the specification file.
  50
  51     :param table: Table to generate.
  52     :param input_data: Data to process.
  53     :type table: pandas.Series
  54     :type input_data: InputData
  55     """
  56
  57     logging.info("  Generating the table {0} ...".
  58                  format(table.get("title", "")))
  59
  60     # Transform the data
  61     data = input_data.filter_data(table)
  62
  63     # Prepare the header of the tables
  64     header = list()
  65     for column in table["columns"]:
  66         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  67
  68     # Generate the data for the table according to the model in the table
  69     # specification
  70     job = table["data"].keys()[0]
  71     build = str(table["data"][job][0])
  72     try:
  73         suites = input_data.suites(job, build)
  74     except KeyError:
  75         logging.error("    No data available. The table will not be generated.")
  76         return
  77
  78     for suite_longname, suite in suites.iteritems():
  79         # Generate data
  80         suite_name = suite["name"]
  81         table_lst = list()
  82         for test in data[job][build].keys():
  83             if data[job][build][test]["parent"] in suite_name:
  84                 row_lst = list()
  85                 for column in table["columns"]:
  86                     try:
  87                         col_data = str(data[job][build][test][column["data"].
  88                                        split(" ")[1]]).replace('"', '""')
  89                         if column["data"].split(" ")[1] in ("vat-history",
  90                                                             "show-run"):
  91                             col_data = replace(col_data, " |br| ", "",
  92                                                maxreplace=1)
  93                             col_data = " |prein| {0} |preout| ".\
  94                                 format(col_data[:-5])
  95                         row_lst.append('"{0}"'.format(col_data))
  96                     except KeyError:
  97                         row_lst.append("No data")
  98                 table_lst.append(row_lst)
  99
 100         # Write the data to file
 101         if table_lst:
 102             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 103                                             table["output-file-ext"])
 104             logging.info("      Writing file: '{}'".format(file_name))
 105             with open(file_name, "w") as file_handler:
 106                 file_handler.write(",".join(header) + "\n")
 107                 for item in table_lst:
 108                     file_handler.write(",".join(item) + "\n")
 109
 110     logging.info("  Done.")
 111
 112
 113 def table_merged_details(table, input_data):
 114     """Generate the table(s) with algorithm: table_merged_details
 115     specified in the specification file.
 116
 117     :param table: Table to generate.
 118     :param input_data: Data to process.
 119     :type table: pandas.Series
 120     :type input_data: InputData
 121     """
 122
 123     logging.info("  Generating the table {0} ...".
 124                  format(table.get("title", "")))
 125
 126     # Transform the data
 127     data = input_data.filter_data(table)
 128     data = input_data.merge_data(data)
 129     data.sort_index(inplace=True)
 130
 131     suites = input_data.filter_data(table, data_set="suites")
 132     suites = input_data.merge_data(suites)
 133
 134     # Prepare the header of the tables
 135     header = list()
 136     for column in table["columns"]:
 137         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 138
 139     for _, suite in suites.iteritems():
 140         # Generate data
 141         suite_name = suite["name"]
 142         table_lst = list()
 143         for test in data.keys():
 144             if data[test]["parent"] in suite_name:
 145                 row_lst = list()
 146                 for column in table["columns"]:
 147                     try:
 148                         col_data = str(data[test][column["data"].
 149                                        split(" ")[1]]).replace('"', '""')
 150                         if column["data"].split(" ")[1] in ("vat-history",
 151                                                             "show-run"):
 152                             col_data = replace(col_data, " |br| ", "",
 153                                                maxreplace=1)
 154                             col_data = " |prein| {0} |preout| ".\
 155                                 format(col_data[:-5])
 156                         row_lst.append('"{0}"'.format(col_data))
 157                     except KeyError:
 158                         row_lst.append("No data")
 159                 table_lst.append(row_lst)
 160
 161         # Write the data to file
 162         if table_lst:
 163             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 164                                             table["output-file-ext"])
 165             logging.info("      Writing file: '{}'".format(file_name))
 166             with open(file_name, "w") as file_handler:
 167                 file_handler.write(",".join(header) + "\n")
 168                 for item in table_lst:
 169                     file_handler.write(",".join(item) + "\n")
 170
 171     logging.info("  Done.")
 172
 173
 174 def table_performance_improvements(table, input_data):
 175     """Generate the table(s) with algorithm: table_performance_improvements
 176     specified in the specification file.
 177
 178     :param table: Table to generate.
 179     :param input_data: Data to process.
 180     :type table: pandas.Series
 181     :type input_data: InputData
 182     """
 183
 184     def _write_line_to_file(file_handler, data):
 185         """Write a line to the .csv file.
 186
 187         :param file_handler: File handler for the csv file. It must be open for
 188          writing text.
 189         :param data: Item to be written to the file.
 190         :type file_handler: BinaryIO
 191         :type data: list
 192         """
 193
 194         line_lst = list()
 195         for item in data:
 196             if isinstance(item["data"], str):
 197                 line_lst.append(item["data"])
 198                 # Remove -?drdisc from the end
 199                 if item["data"].endswith("drdisc"):
 200                     item["data"] = item["data"][:-8]
 201             elif isinstance(item["data"], float):
 202                 line_lst.append("{:.1f}".format(item["data"]))
 203             elif item["data"] is None:
 204                 line_lst.append("")
 205         file_handler.write(",".join(line_lst) + "\n")
 206
 207     logging.info("  Generating the table {0} ...".
 208                  format(table.get("title", "")))
 209
 210     # Read the template
 211     file_name = table.get("template", None)
 212     if file_name:
 213         try:
 214             tmpl = _read_csv_template(file_name)
 215         except PresentationError:
 216             logging.error("  The template '{0}' does not exist. Skipping the "
 217                           "table.".format(file_name))
 218             return None
 219     else:
 220         logging.error("The template is not defined. Skipping the table.")
 221         return None
 222
 223     # Transform the data
 224     data = input_data.filter_data(table)
 225
 226     # Prepare the header of the tables
 227     header = list()
 228     for column in table["columns"]:
 229         header.append(column["title"])
 230
 231     # Generate the data for the table according to the model in the table
 232     # specification
 233     tbl_lst = list()
 234     for tmpl_item in tmpl:
 235         tbl_item = list()
 236         for column in table["columns"]:
 237             cmd = column["data"].split(" ")[0]
 238             args = column["data"].split(" ")[1:]
 239             if cmd == "template":
 240                 try:
 241                     val = float(tmpl_item[int(args[0])])
 242                 except ValueError:
 243                     val = tmpl_item[int(args[0])]
 244                 tbl_item.append({"data": val})
 245             elif cmd == "data":
 246                 jobs = args[0:-1]
 247                 operation = args[-1]
 248                 data_lst = list()
 249                 for job in jobs:
 250                     for build in data[job]:
 251                         try:
 252                             data_lst.append(float(build[tmpl_item[0]]
 253                                                   ["throughput"]["value"]))
 254                         except (KeyError, TypeError):
 255                             # No data, ignore
 256                             continue
 257                 if data_lst:
 258                     tbl_item.append({"data": (eval(operation)(data_lst)) /
 259                                              1000000})
 260                 else:
 261                     tbl_item.append({"data": None})
 262             elif cmd == "operation":
 263                 operation = args[0]
 264                 try:
 265                     nr1 = float(tbl_item[int(args[1])]["data"])
 266                     nr2 = float(tbl_item[int(args[2])]["data"])
 267                     if nr1 and nr2:
 268                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
 269                     else:
 270                         tbl_item.append({"data": None})
 271                 except (IndexError, ValueError, TypeError):
 272                     logging.error("No data for {0}".format(tbl_item[1]["data"]))
 273                     tbl_item.append({"data": None})
 274                     continue
 275             else:
 276                 logging.error("Not supported command {0}. Skipping the table.".
 277                               format(cmd))
 278                 return None
 279         tbl_lst.append(tbl_item)
 280
 281     # Sort the table according to the relative change
 282     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
 283
 284     # Create the tables and write them to the files
 285     file_names = [
 286         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
 287         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
 288         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
 289         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
 290     ]
 291
 292     for file_name in file_names:
 293         logging.info("    Writing the file '{0}'".format(file_name))
 294         with open(file_name, "w") as file_handler:
 295             file_handler.write(",".join(header) + "\n")
 296             for item in tbl_lst:
 297                 if isinstance(item[-1]["data"], float):
 298                     rel_change = round(item[-1]["data"], 1)
 299                 else:
 300                     rel_change = item[-1]["data"]
 301                 if "ndr_top" in file_name \
 302                         and "ndr" in item[1]["data"] \
 303                         and rel_change >= 10.0:
 304                     _write_line_to_file(file_handler, item)
 305                 elif "pdr_top" in file_name \
 306                         and "pdr" in item[1]["data"] \
 307                         and rel_change >= 10.0:
 308                     _write_line_to_file(file_handler, item)
 309                 elif "ndr_low" in file_name \
 310                         and "ndr" in item[1]["data"] \
 311                         and rel_change < 10.0:
 312                     _write_line_to_file(file_handler, item)
 313                 elif "pdr_low" in file_name \
 314                         and "pdr" in item[1]["data"] \
 315                         and rel_change < 10.0:
 316                     _write_line_to_file(file_handler, item)
 317
 318     logging.info("  Done.")
 319
 320
 321 def _read_csv_template(file_name):
 322     """Read the template from a .csv file.
 323
 324     :param file_name: Name / full path / relative path of the file to read.
 325     :type file_name: str
 326     :returns: Data from the template as list (lines) of lists (items on line).
 327     :rtype: list
 328     :raises: PresentationError if it is not possible to read the file.
 329     """
 330
 331     try:
 332         with open(file_name, 'r') as csv_file:
 333             tmpl_data = list()
 334             for line in csv_file:
 335                 tmpl_data.append(line[:-1].split(","))
 336         return tmpl_data
 337     except IOError as err:
 338         raise PresentationError(str(err), level="ERROR")
 339
 340
 341 def table_performance_comparison(table, input_data):
 342     """Generate the table(s) with algorithm: table_performance_comparison
 343     specified in the specification file.
 344
 345     :param table: Table to generate.
 346     :param input_data: Data to process.
 347     :type table: pandas.Series
 348     :type input_data: InputData
 349     """
 350
 351     logging.info("  Generating the table {0} ...".
 352                  format(table.get("title", "")))
 353
 354     # Transform the data
 355     data = input_data.filter_data(table)
 356     logging.info(data)
 357
 358     # Prepare the header of the tables
 359     try:
 360         header = ["Test case",
 361                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 362                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 363                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 364                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 365                   "Change [%]"]
 366         header_str = ",".join(header) + "\n"
 367     except (AttributeError, KeyError) as err:
 368         logging.error("The model is invalid, missing parameter: {0}".
 369                       format(err))
 370         return
 371
 372     # Prepare data to the table:
 373     tbl_dict = dict()
 374     for job, builds in table["reference"]["data"].items():
 375         for build in builds:
 376             for tst_name, tst_data in data[job][str(build)].iteritems():
 377                 if tbl_dict.get(tst_name, None) is None:
 378                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 379                                             "-".join(tst_data["name"].
 380                                                      split("-")[1:]))
 381                     tbl_dict[tst_name] = {"name": name,
 382                                           "ref-data": list(),
 383                                           "cmp-data": list()}
 384                 try:
 385                     tbl_dict[tst_name]["ref-data"].\
 386                         append(tst_data["throughput"]["value"])
 387                 except TypeError:
 388                     pass  # No data in output.xml for this test
 389     logging.info(pformat(tbl_dict))
 390
 391     for job, builds in table["compare"]["data"].items():
 392         for build in builds:
 393             for tst_name, tst_data in data[job][str(build)].iteritems():
 394                 try:
 395                     tbl_dict[tst_name]["cmp-data"].\
 396                         append(tst_data["throughput"]["value"])
 397                 except KeyError:
 398                     pass
 399                 except TypeError:
 400                     tbl_dict.pop(tst_name, None)
 401
 402     tbl_lst = list()
 403     for tst_name in tbl_dict.keys():
 404         item = [tbl_dict[tst_name]["name"], ]
 405         if tbl_dict[tst_name]["ref-data"]:
 406             item.append(round(mean(tbl_dict[tst_name]["ref-data"]) / 1000000,
 407                               2))
 408             item.append(round(stdev(tbl_dict[tst_name]["ref-data"]) / 1000000,
 409                               2))
 410         else:
 411             item.extend([None, None])
 412         if tbl_dict[tst_name]["cmp-data"]:
 413             item.append(round(mean(tbl_dict[tst_name]["cmp-data"]) / 1000000,
 414                               2))
 415             item.append(round(stdev(tbl_dict[tst_name]["cmp-data"]) / 1000000,
 416                               2))
 417         else:
 418             item.extend([None, None])
 419         if item[1] is not None and item[3] is not None:
 420             item.append(int(relative_change(float(item[1]), float(item[3]))))
 421         if len(item) == 6:
 422             tbl_lst.append(item)
 423
 424     # Sort the table according to the relative change
 425     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 426
 427     # Generate tables:
 428     # All tests in csv:
 429     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
 430                                                table["output-file-ext"]),
 431                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
 432                                                table["output-file-ext"]),
 433                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
 434                                                table["output-file-ext"]),
 435                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 436                                                table["output-file-ext"]),
 437                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
 438                                                table["output-file-ext"]),
 439                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
 440                                                table["output-file-ext"])
 441                  ]
 442     for file_name in tbl_names:
 443         logging.info("      Writing file: '{}'".format(file_name))
 444         with open(file_name, "w") as file_handler:
 445             file_handler.write(header_str)
 446             for test in tbl_lst:
 447                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
 448                         file_name.split("-")[-2] in test[0]):  # cores
 449                     test[0] = "-".join(test[0].split("-")[:-1])
 450                     file_handler.write(",".join([str(item) for item in test]) +
 451                                        "\n")
 452
 453     # All tests in txt:
 454     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
 455                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
 456                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
 457                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
 458                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
 459                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
 460                      ]
 461
 462     for i, txt_name in enumerate(tbl_names_txt):
 463         txt_table = None
 464         logging.info("      Writing file: '{}'".format(txt_name))
 465         with open(tbl_names[i], 'rb') as csv_file:
 466             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 467             for row in csv_content:
 468                 if txt_table is None:
 469                     txt_table = prettytable.PrettyTable(row)
 470                 else:
 471                     txt_table.add_row(row)
 472         with open(txt_name, "w") as txt_file:
 473             txt_file.write(str(txt_table))
 474
 475     # Selected tests in csv:
 476     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
 477                                                table["output-file-ext"])
 478     with open(input_file, "r") as in_file:
 479         lines = list()
 480         for line in in_file:
 481             lines.append(line)
 482
 483     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
 484                                                table["output-file-ext"])
 485     logging.info("      Writing file: '{}'".format(output_file))
 486     with open(output_file, "w") as out_file:
 487         out_file.write(header_str)
 488         for i, line in enumerate(lines[1:]):
 489             if i == table["nr-of-tests-shown"]:
 490                 break
 491             out_file.write(line)
 492
 493     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
 494                                                   table["output-file-ext"])
 495     logging.info("      Writing file: '{}'".format(output_file))
 496     with open(output_file, "w") as out_file:
 497         out_file.write(header_str)
 498         for i, line in enumerate(lines[-1:0:-1]):
 499             if i == table["nr-of-tests-shown"]:
 500                 break
 501             out_file.write(line)
 502
 503     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 504                                                table["output-file-ext"])
 505     with open(input_file, "r") as in_file:
 506         lines = list()
 507         for line in in_file:
 508             lines.append(line)
 509
 510     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
 511                                                table["output-file-ext"])
 512     logging.info("      Writing file: '{}'".format(output_file))
 513     with open(output_file, "w") as out_file:
 514         out_file.write(header_str)
 515         for i, line in enumerate(lines[1:]):
 516             if i == table["nr-of-tests-shown"]:
 517                 break
 518             out_file.write(line)
 519
 520     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
 521                                                   table["output-file-ext"])
 522     logging.info("      Writing file: '{}'".format(output_file))
 523     with open(output_file, "w") as out_file:
 524         out_file.write(header_str)
 525         for i, line in enumerate(lines[-1:0:-1]):
 526             if i == table["nr-of-tests-shown"]:
 527                 break
 528             out_file.write(line)