resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import pandas as pd
  21
  22 from string import replace
  23 from collections import OrderedDict
  24 from numpy import nan, isnan
  25 from xml.etree import ElementTree as ET
  26
  27 from errors import PresentationError
  28 from utils import mean, stdev, relative_change, classify_anomalies, \
  29     convert_csv_to_pretty_txt
  30
  31
  32 def generate_tables(spec, data):
  33     """Generate all tables specified in the specification file.
  34
  35     :param spec: Specification read from the specification file.
  36     :param data: Data to process.
  37     :type spec: Specification
  38     :type data: InputData
  39     """
  40
  41     logging.info("Generating the tables ...")
  42     for table in spec.tables:
  43         try:
  44             eval(table["algorithm"])(table, data)
  45         except NameError as err:
  46             logging.error("Probably algorithm '{alg}' is not defined: {err}".
  47                           format(alg=table["algorithm"], err=repr(err)))
  48     logging.info("Done.")
  49
  50
  51 def table_details(table, input_data):
  52     """Generate the table(s) with algorithm: table_detailed_test_results
  53     specified in the specification file.
  54
  55     :param table: Table to generate.
  56     :param input_data: Data to process.
  57     :type table: pandas.Series
  58     :type input_data: InputData
  59     """
  60
  61     logging.info("  Generating the table {0} ...".
  62                  format(table.get("title", "")))
  63
  64     # Transform the data
  65     logging.info("    Creating the data set for the {0} '{1}'.".
  66                  format(table.get("type", ""), table.get("title", "")))
  67     data = input_data.filter_data(table)
  68
  69     # Prepare the header of the tables
  70     header = list()
  71     for column in table["columns"]:
  72         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  73
  74     # Generate the data for the table according to the model in the table
  75     # specification
  76     job = table["data"].keys()[0]
  77     build = str(table["data"][job][0])
  78     try:
  79         suites = input_data.suites(job, build)
  80     except KeyError:
  81         logging.error("    No data available. The table will not be generated.")
  82         return
  83
  84     for suite_longname, suite in suites.iteritems():
  85         # Generate data
  86         suite_name = suite["name"]
  87         table_lst = list()
  88         for test in data[job][build].keys():
  89             if data[job][build][test]["parent"] in suite_name:
  90                 row_lst = list()
  91                 for column in table["columns"]:
  92                     try:
  93                         col_data = str(data[job][build][test][column["data"].
  94                                        split(" ")[1]]).replace('"', '""')
  95                         if column["data"].split(" ")[1] in ("vat-history",
  96                                                             "show-run"):
  97                             col_data = replace(col_data, " |br| ", "",
  98                                                maxreplace=1)
  99                             col_data = " |prein| {0} |preout| ".\
 100                                 format(col_data[:-5])
 101                         row_lst.append('"{0}"'.format(col_data))
 102                     except KeyError:
 103                         row_lst.append("No data")
 104                 table_lst.append(row_lst)
 105
 106         # Write the data to file
 107         if table_lst:
 108             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 109                                             table["output-file-ext"])
 110             logging.info("      Writing file: '{}'".format(file_name))
 111             with open(file_name, "w") as file_handler:
 112                 file_handler.write(",".join(header) + "\n")
 113                 for item in table_lst:
 114                     file_handler.write(",".join(item) + "\n")
 115
 116     logging.info("  Done.")
 117
 118
 119 def table_merged_details(table, input_data):
 120     """Generate the table(s) with algorithm: table_merged_details
 121     specified in the specification file.
 122
 123     :param table: Table to generate.
 124     :param input_data: Data to process.
 125     :type table: pandas.Series
 126     :type input_data: InputData
 127     """
 128
 129     logging.info("  Generating the table {0} ...".
 130                  format(table.get("title", "")))
 131
 132     # Transform the data
 133     logging.info("    Creating the data set for the {0} '{1}'.".
 134                  format(table.get("type", ""), table.get("title", "")))
 135     data = input_data.filter_data(table)
 136     data = input_data.merge_data(data)
 137     data.sort_index(inplace=True)
 138
 139     logging.info("    Creating the data set for the {0} '{1}'.".
 140                  format(table.get("type", ""), table.get("title", "")))
 141     suites = input_data.filter_data(table, data_set="suites")
 142     suites = input_data.merge_data(suites)
 143
 144     # Prepare the header of the tables
 145     header = list()
 146     for column in table["columns"]:
 147         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 148
 149     for _, suite in suites.iteritems():
 150         # Generate data
 151         suite_name = suite["name"]
 152         table_lst = list()
 153         for test in data.keys():
 154             if data[test]["parent"] in suite_name:
 155                 row_lst = list()
 156                 for column in table["columns"]:
 157                     try:
 158                         col_data = str(data[test][column["data"].
 159                                        split(" ")[1]]).replace('"', '""')
 160                         if column["data"].split(" ")[1] in ("vat-history",
 161                                                             "show-run"):
 162                             col_data = replace(col_data, " |br| ", "",
 163                                                maxreplace=1)
 164                             col_data = " |prein| {0} |preout| ".\
 165                                 format(col_data[:-5])
 166                         row_lst.append('"{0}"'.format(col_data))
 167                     except KeyError:
 168                         row_lst.append("No data")
 169                 table_lst.append(row_lst)
 170
 171         # Write the data to file
 172         if table_lst:
 173             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 174                                             table["output-file-ext"])
 175             logging.info("      Writing file: '{}'".format(file_name))
 176             with open(file_name, "w") as file_handler:
 177                 file_handler.write(",".join(header) + "\n")
 178                 for item in table_lst:
 179                     file_handler.write(",".join(item) + "\n")
 180
 181     logging.info("  Done.")
 182
 183
 184 def table_performance_improvements(table, input_data):
 185     """Generate the table(s) with algorithm: table_performance_improvements
 186     specified in the specification file.
 187
 188     :param table: Table to generate.
 189     :param input_data: Data to process.
 190     :type table: pandas.Series
 191     :type input_data: InputData
 192     """
 193
 194     def _write_line_to_file(file_handler, data):
 195         """Write a line to the .csv file.
 196
 197         :param file_handler: File handler for the csv file. It must be open for
 198          writing text.
 199         :param data: Item to be written to the file.
 200         :type file_handler: BinaryIO
 201         :type data: list
 202         """
 203
 204         line_lst = list()
 205         for item in data:
 206             if isinstance(item["data"], str):
 207                 # Remove -?drdisc from the end
 208                 if item["data"].endswith("drdisc"):
 209                     item["data"] = item["data"][:-8]
 210                 line_lst.append(item["data"])
 211             elif isinstance(item["data"], float):
 212                 line_lst.append("{:.1f}".format(item["data"]))
 213             elif item["data"] is None:
 214                 line_lst.append("")
 215         file_handler.write(",".join(line_lst) + "\n")
 216
 217     logging.info("  Generating the table {0} ...".
 218                  format(table.get("title", "")))
 219
 220     # Read the template
 221     file_name = table.get("template", None)
 222     if file_name:
 223         try:
 224             tmpl = _read_csv_template(file_name)
 225         except PresentationError:
 226             logging.error("  The template '{0}' does not exist. Skipping the "
 227                           "table.".format(file_name))
 228             return None
 229     else:
 230         logging.error("The template is not defined. Skipping the table.")
 231         return None
 232
 233     # Transform the data
 234     logging.info("    Creating the data set for the {0} '{1}'.".
 235                  format(table.get("type", ""), table.get("title", "")))
 236     data = input_data.filter_data(table)
 237
 238     # Prepare the header of the tables
 239     header = list()
 240     for column in table["columns"]:
 241         header.append(column["title"])
 242
 243     # Generate the data for the table according to the model in the table
 244     # specification
 245     tbl_lst = list()
 246     for tmpl_item in tmpl:
 247         tbl_item = list()
 248         for column in table["columns"]:
 249             cmd = column["data"].split(" ")[0]
 250             args = column["data"].split(" ")[1:]
 251             if cmd == "template":
 252                 try:
 253                     val = float(tmpl_item[int(args[0])])
 254                 except ValueError:
 255                     val = tmpl_item[int(args[0])]
 256                 tbl_item.append({"data": val})
 257             elif cmd == "data":
 258                 jobs = args[0:-1]
 259                 operation = args[-1]
 260                 data_lst = list()
 261                 for job in jobs:
 262                     for build in data[job]:
 263                         try:
 264                             data_lst.append(float(build[tmpl_item[0]]
 265                                                   ["throughput"]["value"]))
 266                         except (KeyError, TypeError):
 267                             # No data, ignore
 268                             continue
 269                 if data_lst:
 270                     tbl_item.append({"data": (eval(operation)(data_lst)) /
 271                                              1000000})
 272                 else:
 273                     tbl_item.append({"data": None})
 274             elif cmd == "operation":
 275                 operation = args[0]
 276                 try:
 277                     nr1 = float(tbl_item[int(args[1])]["data"])
 278                     nr2 = float(tbl_item[int(args[2])]["data"])
 279                     if nr1 and nr2:
 280                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
 281                     else:
 282                         tbl_item.append({"data": None})
 283                 except (IndexError, ValueError, TypeError):
 284                     logging.error("No data for {0}".format(tbl_item[0]["data"]))
 285                     tbl_item.append({"data": None})
 286                     continue
 287             else:
 288                 logging.error("Not supported command {0}. Skipping the table.".
 289                               format(cmd))
 290                 return None
 291         tbl_lst.append(tbl_item)
 292
 293     # Sort the table according to the relative change
 294     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
 295
 296     # Create the tables and write them to the files
 297     file_names = [
 298         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
 299         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
 300         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
 301         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
 302     ]
 303
 304     for file_name in file_names:
 305         logging.info("    Writing the file '{0}'".format(file_name))
 306         with open(file_name, "w") as file_handler:
 307             file_handler.write(",".join(header) + "\n")
 308             for item in tbl_lst:
 309                 if isinstance(item[-1]["data"], float):
 310                     rel_change = round(item[-1]["data"], 1)
 311                 else:
 312                     rel_change = item[-1]["data"]
 313                 if "ndr_top" in file_name \
 314                         and "ndr" in item[0]["data"] \
 315                         and rel_change >= 10.0:
 316                     _write_line_to_file(file_handler, item)
 317                 elif "pdr_top" in file_name \
 318                         and "pdr" in item[0]["data"] \
 319                         and rel_change >= 10.0:
 320                     _write_line_to_file(file_handler, item)
 321                 elif "ndr_low" in file_name \
 322                         and "ndr" in item[0]["data"] \
 323                         and rel_change < 10.0:
 324                     _write_line_to_file(file_handler, item)
 325                 elif "pdr_low" in file_name \
 326                         and "pdr" in item[0]["data"] \
 327                         and rel_change < 10.0:
 328                     _write_line_to_file(file_handler, item)
 329
 330     logging.info("  Done.")
 331
 332
 333 def _read_csv_template(file_name):
 334     """Read the template from a .csv file.
 335
 336     :param file_name: Name / full path / relative path of the file to read.
 337     :type file_name: str
 338     :returns: Data from the template as list (lines) of lists (items on line).
 339     :rtype: list
 340     :raises: PresentationError if it is not possible to read the file.
 341     """
 342
 343     try:
 344         with open(file_name, 'r') as csv_file:
 345             tmpl_data = list()
 346             for line in csv_file:
 347                 tmpl_data.append(line[:-1].split(","))
 348         return tmpl_data
 349     except IOError as err:
 350         raise PresentationError(str(err), level="ERROR")
 351
 352
 353 def table_performance_comparison(table, input_data):
 354     """Generate the table(s) with algorithm: table_performance_comparison
 355     specified in the specification file.
 356
 357     :param table: Table to generate.
 358     :param input_data: Data to process.
 359     :type table: pandas.Series
 360     :type input_data: InputData
 361     """
 362
 363     logging.info("  Generating the table {0} ...".
 364                  format(table.get("title", "")))
 365
 366     # Transform the data
 367     logging.info("    Creating the data set for the {0} '{1}'.".
 368                  format(table.get("type", ""), table.get("title", "")))
 369     data = input_data.filter_data(table, continue_on_error=True)
 370
 371     # Prepare the header of the tables
 372     try:
 373         header = ["Test case", ]
 374
 375         history = table.get("history", None)
 376         if history:
 377             for item in history:
 378                 header.extend(
 379                     ["{0} Throughput [Mpps]".format(item["title"]),
 380                      "{0} Stdev [Mpps]".format(item["title"])])
 381         header.extend(
 382             ["{0} Throughput [Mpps]".format(table["reference"]["title"]),
 383              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 384              "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 385              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 386              "Change [%]"])
 387         header_str = ",".join(header) + "\n"
 388     except (AttributeError, KeyError) as err:
 389         logging.error("The model is invalid, missing parameter: {0}".
 390                       format(err))
 391         return
 392
 393     # Prepare data to the table:
 394     tbl_dict = dict()
 395     for job, builds in table["reference"]["data"].items():
 396         for build in builds:
 397             for tst_name, tst_data in data[job][str(build)].iteritems():
 398                 if tbl_dict.get(tst_name, None) is None:
 399                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 400                                             "-".join(tst_data["name"].
 401                                                      split("-")[1:]))
 402                     tbl_dict[tst_name] = {"name": name,
 403                                           "ref-data": list(),
 404                                           "cmp-data": list()}
 405                 try:
 406                     tbl_dict[tst_name]["ref-data"].\
 407                         append(tst_data["throughput"]["value"])
 408                 except TypeError:
 409                     pass  # No data in output.xml for this test
 410
 411     for job, builds in table["compare"]["data"].items():
 412         for build in builds:
 413             for tst_name, tst_data in data[job][str(build)].iteritems():
 414                 try:
 415                     tbl_dict[tst_name]["cmp-data"].\
 416                         append(tst_data["throughput"]["value"])
 417                 except KeyError:
 418                     pass
 419                 except TypeError:
 420                     tbl_dict.pop(tst_name, None)
 421     if history:
 422         for item in history:
 423             for job, builds in item["data"].items():
 424                 for build in builds:
 425                     for tst_name, tst_data in data[job][str(build)].iteritems():
 426                         if tbl_dict.get(tst_name, None) is None:
 427                             continue
 428                         if tbl_dict[tst_name].get("history", None) is None:
 429                             tbl_dict[tst_name]["history"] = OrderedDict()
 430                         if tbl_dict[tst_name]["history"].get(item["title"],
 431                                                              None) is None:
 432                             tbl_dict[tst_name]["history"][item["title"]] = \
 433                                 list()
 434                         try:
 435                             tbl_dict[tst_name]["history"][item["title"]].\
 436                                 append(tst_data["throughput"]["value"])
 437                         except (TypeError, KeyError):
 438                             pass
 439
 440     tbl_lst = list()
 441     for tst_name in tbl_dict.keys():
 442         item = [tbl_dict[tst_name]["name"], ]
 443         if history:
 444             if tbl_dict[tst_name].get("history", None) is not None:
 445                 for hist_data in tbl_dict[tst_name]["history"].values():
 446                     if hist_data:
 447                         item.append(round(mean(hist_data) / 1000000, 2))
 448                         item.append(round(stdev(hist_data) / 1000000, 2))
 449                     else:
 450                         item.extend([None, None])
 451             else:
 452                 item.extend([None, None])
 453         data_t = tbl_dict[tst_name]["ref-data"]
 454         if data_t:
 455             item.append(round(mean(data_t) / 1000000, 2))
 456             item.append(round(stdev(data_t) / 1000000, 2))
 457         else:
 458             item.extend([None, None])
 459         data_t = tbl_dict[tst_name]["cmp-data"]
 460         if data_t:
 461             item.append(round(mean(data_t) / 1000000, 2))
 462             item.append(round(stdev(data_t) / 1000000, 2))
 463         else:
 464             item.extend([None, None])
 465         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
 466             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 467         if len(item) == len(header):
 468             tbl_lst.append(item)
 469
 470     # Sort the table according to the relative change
 471     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 472
 473     # Generate tables:
 474     # All tests in csv:
 475     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
 476                                                table["output-file-ext"]),
 477                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
 478                                                table["output-file-ext"]),
 479                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
 480                                                table["output-file-ext"]),
 481                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 482                                                table["output-file-ext"]),
 483                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
 484                                                table["output-file-ext"]),
 485                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
 486                                                table["output-file-ext"])
 487                  ]
 488     for file_name in tbl_names:
 489         logging.info("      Writing file: '{0}'".format(file_name))
 490         with open(file_name, "w") as file_handler:
 491             file_handler.write(header_str)
 492             for test in tbl_lst:
 493                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
 494                         file_name.split("-")[-2] in test[0]):  # cores
 495                     test[0] = "-".join(test[0].split("-")[:-1])
 496                     file_handler.write(",".join([str(item) for item in test]) +
 497                                        "\n")
 498
 499     # All tests in txt:
 500     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
 501                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
 502                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
 503                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
 504                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
 505                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
 506                      ]
 507
 508     for i, txt_name in enumerate(tbl_names_txt):
 509         logging.info("      Writing file: '{0}'".format(txt_name))
 510         convert_csv_to_pretty_txt(tbl_names[i], txt_name)
 511
 512     # Selected tests in csv:
 513     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
 514                                                table["output-file-ext"])
 515     with open(input_file, "r") as in_file:
 516         lines = list()
 517         for line in in_file:
 518             lines.append(line)
 519
 520     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
 521                                                table["output-file-ext"])
 522     logging.info("      Writing file: '{0}'".format(output_file))
 523     with open(output_file, "w") as out_file:
 524         out_file.write(header_str)
 525         for i, line in enumerate(lines[1:]):
 526             if i == table["nr-of-tests-shown"]:
 527                 break
 528             out_file.write(line)
 529
 530     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
 531                                                   table["output-file-ext"])
 532     logging.info("      Writing file: '{0}'".format(output_file))
 533     with open(output_file, "w") as out_file:
 534         out_file.write(header_str)
 535         for i, line in enumerate(lines[-1:0:-1]):
 536             if i == table["nr-of-tests-shown"]:
 537                 break
 538             out_file.write(line)
 539
 540     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 541                                                table["output-file-ext"])
 542     with open(input_file, "r") as in_file:
 543         lines = list()
 544         for line in in_file:
 545             lines.append(line)
 546
 547     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
 548                                                table["output-file-ext"])
 549     logging.info("      Writing file: '{0}'".format(output_file))
 550     with open(output_file, "w") as out_file:
 551         out_file.write(header_str)
 552         for i, line in enumerate(lines[1:]):
 553             if i == table["nr-of-tests-shown"]:
 554                 break
 555             out_file.write(line)
 556
 557     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
 558                                                   table["output-file-ext"])
 559     logging.info("      Writing file: '{0}'".format(output_file))
 560     with open(output_file, "w") as out_file:
 561         out_file.write(header_str)
 562         for i, line in enumerate(lines[-1:0:-1]):
 563             if i == table["nr-of-tests-shown"]:
 564                 break
 565             out_file.write(line)
 566
 567
 568 def table_performance_comparison_mrr(table, input_data):
 569     """Generate the table(s) with algorithm: table_performance_comparison_mrr
 570     specified in the specification file.
 571
 572     :param table: Table to generate.
 573     :param input_data: Data to process.
 574     :type table: pandas.Series
 575     :type input_data: InputData
 576     """
 577
 578     logging.info("  Generating the table {0} ...".
 579                  format(table.get("title", "")))
 580
 581     # Transform the data
 582     logging.info("    Creating the data set for the {0} '{1}'.".
 583                  format(table.get("type", ""), table.get("title", "")))
 584     data = input_data.filter_data(table, continue_on_error=True)
 585
 586     # Prepare the header of the tables
 587     try:
 588         header = ["Test case",
 589                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 590                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 591                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 592                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 593                   "Change [%]"]
 594         header_str = ",".join(header) + "\n"
 595     except (AttributeError, KeyError) as err:
 596         logging.error("The model is invalid, missing parameter: {0}".
 597                       format(err))
 598         return
 599
 600     # Prepare data to the table:
 601     tbl_dict = dict()
 602     for job, builds in table["reference"]["data"].items():
 603         for build in builds:
 604             for tst_name, tst_data in data[job][str(build)].iteritems():
 605                 if tbl_dict.get(tst_name, None) is None:
 606                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 607                                             "-".join(tst_data["name"].
 608                                                      split("-")[1:]))
 609                     tbl_dict[tst_name] = {"name": name,
 610                                           "ref-data": list(),
 611                                           "cmp-data": list()}
 612                 try:
 613                     tbl_dict[tst_name]["ref-data"].\
 614                         append(tst_data["result"]["throughput"])
 615                 except TypeError:
 616                     pass  # No data in output.xml for this test
 617
 618     for job, builds in table["compare"]["data"].items():
 619         for build in builds:
 620             for tst_name, tst_data in data[job][str(build)].iteritems():
 621                 try:
 622                     tbl_dict[tst_name]["cmp-data"].\
 623                         append(tst_data["result"]["throughput"])
 624                 except KeyError:
 625                     pass
 626                 except TypeError:
 627                     tbl_dict.pop(tst_name, None)
 628
 629     tbl_lst = list()
 630     for tst_name in tbl_dict.keys():
 631         item = [tbl_dict[tst_name]["name"], ]
 632         data_t = tbl_dict[tst_name]["ref-data"]
 633         if data_t:
 634             item.append(round(mean(data_t) / 1000000, 2))
 635             item.append(round(stdev(data_t) / 1000000, 2))
 636         else:
 637             item.extend([None, None])
 638         data_t = tbl_dict[tst_name]["cmp-data"]
 639         if data_t:
 640             item.append(round(mean(data_t) / 1000000, 2))
 641             item.append(round(stdev(data_t) / 1000000, 2))
 642         else:
 643             item.extend([None, None])
 644         if item[1] is not None and item[3] is not None and item[1] != 0:
 645             item.append(int(relative_change(float(item[1]), float(item[3]))))
 646         if len(item) == 6:
 647             tbl_lst.append(item)
 648
 649     # Sort the table according to the relative change
 650     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 651
 652     # Generate tables:
 653     # All tests in csv:
 654     tbl_names = ["{0}-1t1c-full{1}".format(table["output-file"],
 655                                            table["output-file-ext"]),
 656                  "{0}-2t2c-full{1}".format(table["output-file"],
 657                                            table["output-file-ext"]),
 658                  "{0}-4t4c-full{1}".format(table["output-file"],
 659                                            table["output-file-ext"])
 660                  ]
 661     for file_name in tbl_names:
 662         logging.info("      Writing file: '{0}'".format(file_name))
 663         with open(file_name, "w") as file_handler:
 664             file_handler.write(header_str)
 665             for test in tbl_lst:
 666                 if file_name.split("-")[-2] in test[0]:  # cores
 667                     test[0] = "-".join(test[0].split("-")[:-1])
 668                     file_handler.write(",".join([str(item) for item in test]) +
 669                                        "\n")
 670
 671     # All tests in txt:
 672     tbl_names_txt = ["{0}-1t1c-full.txt".format(table["output-file"]),
 673                      "{0}-2t2c-full.txt".format(table["output-file"]),
 674                      "{0}-4t4c-full.txt".format(table["output-file"])
 675                      ]
 676
 677     for i, txt_name in enumerate(tbl_names_txt):
 678         logging.info("      Writing file: '{0}'".format(txt_name))
 679         convert_csv_to_pretty_txt(tbl_names[i], txt_name)
 680
 681
 682 def table_performance_trending_dashboard(table, input_data):
 683     """Generate the table(s) with algorithm:
 684     table_performance_trending_dashboard
 685     specified in the specification file.
 686
 687     :param table: Table to generate.
 688     :param input_data: Data to process.
 689     :type table: pandas.Series
 690     :type input_data: InputData
 691     """
 692
 693     logging.info("  Generating the table {0} ...".
 694                  format(table.get("title", "")))
 695
 696     # Transform the data
 697     logging.info("    Creating the data set for the {0} '{1}'.".
 698                  format(table.get("type", ""), table.get("title", "")))
 699     data = input_data.filter_data(table, continue_on_error=True)
 700
 701     # Prepare the header of the tables
 702     header = ["Test Case",
 703               "Trend [Mpps]",
 704               "Short-Term Change [%]",
 705               "Long-Term Change [%]",
 706               "Regressions [#]",
 707               "Progressions [#]"
 708               ]
 709     header_str = ",".join(header) + "\n"
 710
 711     # Prepare data to the table:
 712     tbl_dict = dict()
 713     for job, builds in table["data"].items():
 714         for build in builds:
 715             for tst_name, tst_data in data[job][str(build)].iteritems():
 716                 if tst_name.lower() in table["ignore-list"]:
 717                     continue
 718                 if tbl_dict.get(tst_name, None) is None:
 719                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 720                                             "-".join(tst_data["name"].
 721                                                      split("-")[1:]))
 722                     tbl_dict[tst_name] = {"name": name,
 723                                           "data": OrderedDict()}
 724                 try:
 725                     tbl_dict[tst_name]["data"][str(build)] =  \
 726                         tst_data["result"]["throughput"]
 727                 except (TypeError, KeyError):
 728                     pass  # No data in output.xml for this test
 729
 730     tbl_lst = list()
 731     for tst_name in tbl_dict.keys():
 732         if len(tbl_dict[tst_name]["data"]) < 2:
 733             continue
 734
 735         data_t = pd.Series(tbl_dict[tst_name]["data"])
 736
 737         classification_lst, avgs = classify_anomalies(data_t)
 738
 739         win_size = min(data_t.size, table["window"])
 740         long_win_size = min(data_t.size, table["long-trend-window"])
 741         try:
 742             max_long_avg = max(
 743                 [x for x in avgs[-long_win_size:-win_size]
 744                  if not isnan(x)])
 745         except ValueError:
 746             max_long_avg = nan
 747         last_avg = avgs[-1]
 748         avg_week_ago = avgs[max(-win_size, -len(avgs))]
 749
 750         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
 751             rel_change_last = nan
 752         else:
 753             rel_change_last = round(
 754                 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
 755
 756         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
 757             rel_change_long = nan
 758         else:
 759             rel_change_long = round(
 760                 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
 761
 762         if classification_lst:
 763             if isnan(rel_change_last) and isnan(rel_change_long):
 764                 continue
 765             tbl_lst.append(
 766                 [tbl_dict[tst_name]["name"],
 767                  '-' if isnan(last_avg) else
 768                  round(last_avg / 1000000, 2),
 769                  '-' if isnan(rel_change_last) else rel_change_last,
 770                  '-' if isnan(rel_change_long) else rel_change_long,
 771                  classification_lst[-win_size:].count("regression"),
 772                  classification_lst[-win_size:].count("progression")])
 773
 774     tbl_lst.sort(key=lambda rel: rel[0])
 775
 776     tbl_sorted = list()
 777     for nrr in range(table["window"], -1, -1):
 778         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
 779         for nrp in range(table["window"], -1, -1):
 780             tbl_out = [item for item in tbl_reg if item[5] == nrp]
 781             tbl_out.sort(key=lambda rel: rel[2])
 782             tbl_sorted.extend(tbl_out)
 783
 784     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
 785
 786     logging.info("    Writing file: '{0}'".format(file_name))
 787     with open(file_name, "w") as file_handler:
 788         file_handler.write(header_str)
 789         for test in tbl_sorted:
 790             file_handler.write(",".join([str(item) for item in test]) + '\n')
 791
 792     txt_file_name = "{0}.txt".format(table["output-file"])
 793     logging.info("    Writing file: '{0}'".format(txt_file_name))
 794     convert_csv_to_pretty_txt(file_name, txt_file_name)
 795
 796
 797 def _generate_url(base, test_name):
 798     """Generate URL to a trending plot from the name of the test case.
 799
 800     :param base: The base part of URL common to all test cases.
 801     :param test_name: The name of the test case.
 802     :type base: str
 803     :type test_name: str
 804     :returns: The URL to the plot with the trending data for the given test
 805         case.
 806     :rtype str
 807     """
 808
 809     url = base
 810     file_name = ""
 811     anchor = "#"
 812     feature = ""
 813
 814     if "lbdpdk" in test_name or "lbvpp" in test_name:
 815         file_name = "link_bonding.html"
 816
 817     elif "testpmd" in test_name or "l3fwd" in test_name:
 818         file_name = "dpdk.html"
 819
 820     elif "memif" in test_name:
 821         file_name = "container_memif.html"
 822
 823     elif "srv6" in test_name:
 824         file_name = "srv6.html"
 825
 826     elif "vhost" in test_name:
 827         if "l2xcbase" in test_name or "l2bdbasemaclrn" in test_name:
 828             file_name = "vm_vhost_l2.html"
 829         elif "ip4base" in test_name:
 830             file_name = "vm_vhost_ip4.html"
 831
 832     elif "ipsec" in test_name:
 833         file_name = "ipsec.html"
 834
 835     elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name:
 836         file_name = "ip4_tunnels.html"
 837
 838     elif "ip4base" in test_name or "ip4scale" in test_name:
 839         file_name = "ip4.html"
 840         if "iacl" in test_name or "snat" in test_name or "cop" in test_name:
 841             feature = "-features"
 842
 843     elif "ip6base" in test_name or "ip6scale" in test_name:
 844         file_name = "ip6.html"
 845
 846     elif "l2xcbase" in test_name or "l2xcscale" in test_name \
 847             or "l2bdbasemaclrn" in test_name or "l2bdscale" in test_name \
 848             or "l2dbbasemaclrn" in test_name or "l2dbscale" in test_name:
 849         file_name = "l2.html"
 850         if "iacl" in test_name:
 851             feature = "-features"
 852
 853     if "x520" in test_name:
 854         anchor += "x520-"
 855     elif "x710" in test_name:
 856         anchor += "x710-"
 857     elif "xl710" in test_name:
 858         anchor += "xl710-"
 859
 860     if "64b" in test_name:
 861         anchor += "64b-"
 862     elif "78b" in test_name:
 863         anchor += "78b-"
 864     elif "imix" in test_name:
 865         anchor += "imix-"
 866     elif "9000b" in test_name:
 867         anchor += "9000b-"
 868     elif "1518" in test_name:
 869         anchor += "1518b-"
 870
 871     if "1t1c" in test_name:
 872         anchor += "1t1c"
 873     elif "2t2c" in test_name:
 874         anchor += "2t2c"
 875     elif "4t4c" in test_name:
 876         anchor += "4t4c"
 877
 878     return url + file_name + anchor + feature
 879
 880
 881 def table_performance_trending_dashboard_html(table, input_data):
 882     """Generate the table(s) with algorithm:
 883     table_performance_trending_dashboard_html specified in the specification
 884     file.
 885
 886     :param table: Table to generate.
 887     :param input_data: Data to process.
 888     :type table: pandas.Series
 889     :type input_data: InputData
 890     """
 891
 892     logging.info("  Generating the table {0} ...".
 893                  format(table.get("title", "")))
 894
 895     try:
 896         with open(table["input-file"], 'rb') as csv_file:
 897             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 898             csv_lst = [item for item in csv_content]
 899     except KeyError:
 900         logging.warning("The input file is not defined.")
 901         return
 902     except csv.Error as err:
 903         logging.warning("Not possible to process the file '{0}'.\n{1}".
 904                         format(table["input-file"], err))
 905         return
 906
 907     # Table:
 908     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
 909
 910     # Table header:
 911     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
 912     for idx, item in enumerate(csv_lst[0]):
 913         alignment = "left" if idx == 0 else "center"
 914         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
 915         th.text = item
 916
 917     # Rows:
 918     colors = {"regression": ("#ffcccc", "#ff9999"),
 919               "progression": ("#c6ecc6", "#9fdf9f"),
 920               "normal": ("#e9f1fb", "#d4e4f7")}
 921     for r_idx, row in enumerate(csv_lst[1:]):
 922         if int(row[4]):
 923             color = "regression"
 924         elif int(row[5]):
 925             color = "progression"
 926         else:
 927             color = "normal"
 928         background = colors[color][r_idx % 2]
 929         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
 930
 931         # Columns:
 932         for c_idx, item in enumerate(row):
 933             alignment = "left" if c_idx == 0 else "center"
 934             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
 935             # Name:
 936             if c_idx == 0:
 937                 url = _generate_url("../trending/", item)
 938                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
 939                 ref.text = item
 940             else:
 941                 td.text = item
 942     try:
 943         with open(table["output-file"], 'w') as html_file:
 944             logging.info("    Writing file: '{0}'".format(table["output-file"]))
 945             html_file.write(".. raw:: html\n\n\t")
 946             html_file.write(ET.tostring(dashboard))
 947             html_file.write("\n\t<p><br><br></p>\n")
 948     except KeyError:
 949         logging.warning("The output file is not defined.")
 950         return
 951
 952
 953 def table_failed_tests(table, input_data):
 954     """Generate the table(s) with algorithm: table_failed_tests
 955     specified in the specification file.
 956
 957     :param table: Table to generate.
 958     :param input_data: Data to process.
 959     :type table: pandas.Series
 960     :type input_data: InputData
 961     """
 962
 963     logging.info("  Generating the table {0} ...".
 964                  format(table.get("title", "")))
 965
 966     # Transform the data
 967     logging.info("    Creating the data set for the {0} '{1}'.".
 968                  format(table.get("type", ""), table.get("title", "")))
 969     data = input_data.filter_data(table, continue_on_error=True)
 970
 971     # Prepare the header of the tables
 972     header = ["Test Case",
 973               "Failures [#]",
 974               "Last Failure [Time]",
 975               "Last Failure [VPP-Build-Id]",
 976               "Last Failure [CSIT-Job-Build-Id]"]
 977
 978     # Generate the data for the table according to the model in the table
 979     # specification
 980     tbl_dict = dict()
 981     for job, builds in table["data"].items():
 982         for build in builds:
 983             build = str(build)
 984             for tst_name, tst_data in data[job][build].iteritems():
 985                 if tst_name.lower() in table["ignore-list"]:
 986                     continue
 987                 if tbl_dict.get(tst_name, None) is None:
 988                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 989                                             "-".join(tst_data["name"].
 990                                                      split("-")[1:]))
 991                     tbl_dict[tst_name] = {"name": name,
 992                                           "data": OrderedDict()}
 993                 try:
 994                     tbl_dict[tst_name]["data"][build] = (
 995                         tst_data["status"],
 996                         input_data.metadata(job, build).get("generated", ""),
 997                         input_data.metadata(job, build).get("version", ""),
 998                         build)
 999                 except (TypeError, KeyError):
1000                     pass  # No data in output.xml for this test
1001
1002     tbl_lst = list()
1003     for tst_data in tbl_dict.values():
1004         win_size = min(len(tst_data["data"]), table["window"])
1005         fails_nr = 0
1006         for val in tst_data["data"].values()[-win_size:]:
1007             if val[0] == "FAIL":
1008                 fails_nr += 1
1009                 fails_last_date = val[1]
1010                 fails_last_vpp = val[2]
1011                 fails_last_csit = val[3]
1012         if fails_nr:
1013             tbl_lst.append([tst_data["name"],
1014                             fails_nr,
1015                             fails_last_date,
1016                             fails_last_vpp,
1017                             "mrr-daily-build-{0}".format(fails_last_csit)])
1018
1019     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1020     tbl_sorted = list()
1021     for nrf in range(table["window"], -1, -1):
1022         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1023         tbl_sorted.extend(tbl_fails)
1024     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1025
1026     logging.info("    Writing file: '{0}'".format(file_name))
1027     with open(file_name, "w") as file_handler:
1028         file_handler.write(",".join(header) + "\n")
1029         for test in tbl_sorted:
1030             file_handler.write(",".join([str(item) for item in test]) + '\n')
1031
1032     txt_file_name = "{0}.txt".format(table["output-file"])
1033     logging.info("    Writing file: '{0}'".format(txt_file_name))
1034     convert_csv_to_pretty_txt(file_name, txt_file_name)
1035
1036
1037 def table_failed_tests_html(table, input_data):
1038     """Generate the table(s) with algorithm: table_failed_tests_html
1039     specified in the specification file.
1040
1041     :param table: Table to generate.
1042     :param input_data: Data to process.
1043     :type table: pandas.Series
1044     :type input_data: InputData
1045     """
1046
1047     logging.info("  Generating the table {0} ...".
1048                  format(table.get("title", "")))
1049
1050     try:
1051         with open(table["input-file"], 'rb') as csv_file:
1052             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
1053             csv_lst = [item for item in csv_content]
1054     except KeyError:
1055         logging.warning("The input file is not defined.")
1056         return
1057     except csv.Error as err:
1058         logging.warning("Not possible to process the file '{0}'.\n{1}".
1059                         format(table["input-file"], err))
1060         return
1061
1062     # Table:
1063     failed_tests = ET.Element("table", attrib=dict(width="100%", border='0'))
1064
1065     # Table header:
1066     tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor="#7eade7"))
1067     for idx, item in enumerate(csv_lst[0]):
1068         alignment = "left" if idx == 0 else "center"
1069         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
1070         th.text = item
1071
1072     # Rows:
1073     colors = ("#e9f1fb", "#d4e4f7")
1074     for r_idx, row in enumerate(csv_lst[1:]):
1075         background = colors[r_idx % 2]
1076         tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor=background))
1077
1078         # Columns:
1079         for c_idx, item in enumerate(row):
1080             alignment = "left" if c_idx == 0 else "center"
1081             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1082             # Name:
1083             if c_idx == 0:
1084                 url = _generate_url("../trending/", item)
1085                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1086                 ref.text = item
1087             else:
1088                 td.text = item
1089     try:
1090         with open(table["output-file"], 'w') as html_file:
1091             logging.info("    Writing file: '{0}'".format(table["output-file"]))
1092             html_file.write(".. raw:: html\n\n\t")
1093             html_file.write(ET.tostring(failed_tests))
1094             html_file.write("\n\t<p><br><br></p>\n")
1095     except KeyError:
1096         logging.warning("The output file is not defined.")
1097         return