resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from string import replace
  23 from collections import OrderedDict
  24 from numpy import nan, isnan
  25 from xml.etree import ElementTree as ET
  26 from datetime import datetime as dt
  27 from datetime import timedelta
  28
  29 from utils import mean, stdev, relative_change, classify_anomalies, \
  30     convert_csv_to_pretty_txt, relative_change_stdev
  31
  32
  33 REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*')
  34
  35
  36 def generate_tables(spec, data):
  37     """Generate all tables specified in the specification file.
  38
  39     :param spec: Specification read from the specification file.
  40     :param data: Data to process.
  41     :type spec: Specification
  42     :type data: InputData
  43     """
  44
  45     logging.info("Generating the tables ...")
  46     for table in spec.tables:
  47         try:
  48             eval(table["algorithm"])(table, data)
  49         except NameError as err:
  50             logging.error("Probably algorithm '{alg}' is not defined: {err}".
  51                           format(alg=table["algorithm"], err=repr(err)))
  52     logging.info("Done.")
  53
  54
  55 def table_details(table, input_data):
  56     """Generate the table(s) with algorithm: table_detailed_test_results
  57     specified in the specification file.
  58
  59     :param table: Table to generate.
  60     :param input_data: Data to process.
  61     :type table: pandas.Series
  62     :type input_data: InputData
  63     """
  64
  65     logging.info("  Generating the table {0} ...".
  66                  format(table.get("title", "")))
  67
  68     # Transform the data
  69     logging.info("    Creating the data set for the {0} '{1}'.".
  70                  format(table.get("type", ""), table.get("title", "")))
  71     data = input_data.filter_data(table)
  72
  73     # Prepare the header of the tables
  74     header = list()
  75     for column in table["columns"]:
  76         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  77
  78     # Generate the data for the table according to the model in the table
  79     # specification
  80     job = table["data"].keys()[0]
  81     build = str(table["data"][job][0])
  82     try:
  83         suites = input_data.suites(job, build)
  84     except KeyError:
  85         logging.error("    No data available. The table will not be generated.")
  86         return
  87
  88     for suite_longname, suite in suites.iteritems():
  89         # Generate data
  90         suite_name = suite["name"]
  91         table_lst = list()
  92         for test in data[job][build].keys():
  93             if data[job][build][test]["parent"] in suite_name:
  94                 row_lst = list()
  95                 for column in table["columns"]:
  96                     try:
  97                         col_data = str(data[job][build][test][column["data"].
  98                                        split(" ")[1]]).replace('"', '""')
  99                         if column["data"].split(" ")[1] in ("conf-history",
 100                                                             "show-run"):
 101                             col_data = replace(col_data, " |br| ", "",
 102                                                maxreplace=1)
 103                             col_data = " |prein| {0} |preout| ".\
 104                                 format(col_data[:-5])
 105                         row_lst.append('"{0}"'.format(col_data))
 106                     except KeyError:
 107                         row_lst.append("No data")
 108                 table_lst.append(row_lst)
 109
 110         # Write the data to file
 111         if table_lst:
 112             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 113                                             table["output-file-ext"])
 114             logging.info("      Writing file: '{}'".format(file_name))
 115             with open(file_name, "w") as file_handler:
 116                 file_handler.write(",".join(header) + "\n")
 117                 for item in table_lst:
 118                     file_handler.write(",".join(item) + "\n")
 119
 120     logging.info("  Done.")
 121
 122
 123 def table_merged_details(table, input_data):
 124     """Generate the table(s) with algorithm: table_merged_details
 125     specified in the specification file.
 126
 127     :param table: Table to generate.
 128     :param input_data: Data to process.
 129     :type table: pandas.Series
 130     :type input_data: InputData
 131     """
 132
 133     logging.info("  Generating the table {0} ...".
 134                  format(table.get("title", "")))
 135
 136     # Transform the data
 137     logging.info("    Creating the data set for the {0} '{1}'.".
 138                  format(table.get("type", ""), table.get("title", "")))
 139     data = input_data.filter_data(table)
 140     data = input_data.merge_data(data)
 141     data.sort_index(inplace=True)
 142
 143     logging.info("    Creating the data set for the {0} '{1}'.".
 144                  format(table.get("type", ""), table.get("title", "")))
 145     suites = input_data.filter_data(table, data_set="suites")
 146     suites = input_data.merge_data(suites)
 147
 148     # Prepare the header of the tables
 149     header = list()
 150     for column in table["columns"]:
 151         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 152
 153     for _, suite in suites.iteritems():
 154         # Generate data
 155         suite_name = suite["name"]
 156         table_lst = list()
 157         for test in data.keys():
 158             if data[test]["parent"] in suite_name:
 159                 row_lst = list()
 160                 for column in table["columns"]:
 161                     try:
 162                         col_data = str(data[test][column["data"].
 163                                        split(" ")[1]]).replace('"', '""')
 164                         col_data = replace(col_data, "No Data",
 165                                            "Not Captured     ")
 166                         if column["data"].split(" ")[1] in ("conf-history",
 167                                                             "show-run"):
 168                             col_data = replace(col_data, " |br| ", "",
 169                                                maxreplace=1)
 170                             col_data = " |prein| {0} |preout| ".\
 171                                 format(col_data[:-5])
 172                         row_lst.append('"{0}"'.format(col_data))
 173                     except KeyError:
 174                         row_lst.append('"Not captured"')
 175                 table_lst.append(row_lst)
 176
 177         # Write the data to file
 178         if table_lst:
 179             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 180                                             table["output-file-ext"])
 181             logging.info("      Writing file: '{}'".format(file_name))
 182             with open(file_name, "w") as file_handler:
 183                 file_handler.write(",".join(header) + "\n")
 184                 for item in table_lst:
 185                     file_handler.write(",".join(item) + "\n")
 186
 187     logging.info("  Done.")
 188
 189
 190 def table_performance_comparison(table, input_data):
 191     """Generate the table(s) with algorithm: table_performance_comparison
 192     specified in the specification file.
 193
 194     :param table: Table to generate.
 195     :param input_data: Data to process.
 196     :type table: pandas.Series
 197     :type input_data: InputData
 198     """
 199
 200     logging.info("  Generating the table {0} ...".
 201                  format(table.get("title", "")))
 202
 203     # Transform the data
 204     logging.info("    Creating the data set for the {0} '{1}'.".
 205                  format(table.get("type", ""), table.get("title", "")))
 206     data = input_data.filter_data(table, continue_on_error=True)
 207
 208     # Prepare the header of the tables
 209     try:
 210         header = ["Test case", ]
 211
 212         if table["include-tests"] == "MRR":
 213             hdr_param = "Receive Rate"
 214         else:
 215             hdr_param = "Throughput"
 216
 217         history = table.get("history", None)
 218         if history:
 219             for item in history:
 220                 header.extend(
 221                     ["{0} {1} [Mpps]".format(item["title"], hdr_param),
 222                      "{0} Stdev [Mpps]".format(item["title"])])
 223         header.extend(
 224             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
 225              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 226              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
 227              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 228              "Delta [%]"])
 229         header_str = ",".join(header) + "\n"
 230     except (AttributeError, KeyError) as err:
 231         logging.error("The model is invalid, missing parameter: {0}".
 232                       format(err))
 233         return
 234
 235     # Prepare data to the table:
 236     tbl_dict = dict()
 237     for job, builds in table["reference"]["data"].items():
 238         for build in builds:
 239             for tst_name, tst_data in data[job][str(build)].iteritems():
 240                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
 241                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
 242                     replace("-ndrdisc", "").replace("-pdr", "").\
 243                     replace("-ndr", "").\
 244                     replace("1t1c", "1c").replace("2t1c", "1c").\
 245                     replace("2t2c", "2c").replace("4t2c", "2c").\
 246                     replace("4t4c", "4c").replace("8t4c", "4c")
 247                 if "across topologies" in table["title"].lower():
 248                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
 249                 if tbl_dict.get(tst_name_mod, None) is None:
 250                     groups = re.search(REGEX_NIC, tst_data["parent"])
 251                     nic = groups.group(0) if groups else ""
 252                     name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
 253                                                           split("-")[:-1]))
 254                     if "across testbeds" in table["title"].lower() or \
 255                             "across topologies" in table["title"].lower():
 256                         name = name.\
 257                             replace("1t1c", "1c").replace("2t1c", "1c").\
 258                             replace("2t2c", "2c").replace("4t2c", "2c").\
 259                             replace("4t4c", "4c").replace("8t4c", "4c")
 260                     tbl_dict[tst_name_mod] = {"name": name,
 261                                               "ref-data": list(),
 262                                               "cmp-data": list()}
 263                 try:
 264                     # TODO: Re-work when NDRPDRDISC tests are not used
 265                     if table["include-tests"] == "MRR":
 266                         tbl_dict[tst_name_mod]["ref-data"]. \
 267                             append(tst_data["result"]["receive-rate"].avg)
 268                     elif table["include-tests"] == "PDR":
 269                         if tst_data["type"] == "PDR":
 270                             tbl_dict[tst_name_mod]["ref-data"]. \
 271                                 append(tst_data["throughput"]["value"])
 272                         elif tst_data["type"] == "NDRPDR":
 273                             tbl_dict[tst_name_mod]["ref-data"].append(
 274                                 tst_data["throughput"]["PDR"]["LOWER"])
 275                     elif table["include-tests"] == "NDR":
 276                         if tst_data["type"] == "NDR":
 277                             tbl_dict[tst_name_mod]["ref-data"]. \
 278                                 append(tst_data["throughput"]["value"])
 279                         elif tst_data["type"] == "NDRPDR":
 280                             tbl_dict[tst_name_mod]["ref-data"].append(
 281                                 tst_data["throughput"]["NDR"]["LOWER"])
 282                     else:
 283                         continue
 284                 except TypeError:
 285                     pass  # No data in output.xml for this test
 286
 287     for job, builds in table["compare"]["data"].items():
 288         for build in builds:
 289             for tst_name, tst_data in data[job][str(build)].iteritems():
 290                 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
 291                     replace("-ndrpdr", "").replace("-pdrdisc", ""). \
 292                     replace("-ndrdisc", "").replace("-pdr", ""). \
 293                     replace("-ndr", "").\
 294                     replace("1t1c", "1c").replace("2t1c", "1c").\
 295                     replace("2t2c", "2c").replace("4t2c", "2c").\
 296                     replace("4t4c", "4c").replace("8t4c", "4c")
 297                 if "across topologies" in table["title"].lower():
 298                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
 299                 try:
 300                     # TODO: Re-work when NDRPDRDISC tests are not used
 301                     if table["include-tests"] == "MRR":
 302                         tbl_dict[tst_name_mod]["cmp-data"]. \
 303                             append(tst_data["result"]["receive-rate"].avg)
 304                     elif table["include-tests"] == "PDR":
 305                         if tst_data["type"] == "PDR":
 306                             tbl_dict[tst_name_mod]["cmp-data"]. \
 307                                 append(tst_data["throughput"]["value"])
 308                         elif tst_data["type"] == "NDRPDR":
 309                             tbl_dict[tst_name_mod]["cmp-data"].append(
 310                                 tst_data["throughput"]["PDR"]["LOWER"])
 311                     elif table["include-tests"] == "NDR":
 312                         if tst_data["type"] == "NDR":
 313                             tbl_dict[tst_name_mod]["cmp-data"]. \
 314                                 append(tst_data["throughput"]["value"])
 315                         elif tst_data["type"] == "NDRPDR":
 316                             tbl_dict[tst_name_mod]["cmp-data"].append(
 317                                 tst_data["throughput"]["NDR"]["LOWER"])
 318                     else:
 319                         continue
 320                 except KeyError:
 321                     pass
 322                 except TypeError:
 323                     tbl_dict.pop(tst_name_mod, None)
 324     if history:
 325         for item in history:
 326             for job, builds in item["data"].items():
 327                 for build in builds:
 328                     for tst_name, tst_data in data[job][str(build)].iteritems():
 329                         tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
 330                             replace("-ndrpdr", "").replace("-pdrdisc", ""). \
 331                             replace("-ndrdisc", "").replace("-pdr", ""). \
 332                             replace("-ndr", "").\
 333                             replace("1t1c", "1c").replace("2t1c", "1c").\
 334                             replace("2t2c", "2c").replace("4t2c", "2c").\
 335                             replace("4t4c", "4c").replace("8t4c", "4c")
 336                         if "across topologies" in table["title"].lower():
 337                             tst_name_mod = tst_name_mod.replace("2n1l-", "")
 338                         if tbl_dict.get(tst_name_mod, None) is None:
 339                             continue
 340                         if tbl_dict[tst_name_mod].get("history", None) is None:
 341                             tbl_dict[tst_name_mod]["history"] = OrderedDict()
 342                         if tbl_dict[tst_name_mod]["history"].get(item["title"],
 343                                                              None) is None:
 344                             tbl_dict[tst_name_mod]["history"][item["title"]] = \
 345                                 list()
 346                         try:
 347                             # TODO: Re-work when NDRPDRDISC tests are not used
 348                             if table["include-tests"] == "MRR":
 349                                 tbl_dict[tst_name_mod]["history"][item["title"
 350                                 ]].append(tst_data["result"]["receive-rate"].
 351                                           avg)
 352                             elif table["include-tests"] == "PDR":
 353                                 if tst_data["type"] == "PDR":
 354                                     tbl_dict[tst_name_mod]["history"][
 355                                         item["title"]].\
 356                                         append(tst_data["throughput"]["value"])
 357                                 elif tst_data["type"] == "NDRPDR":
 358                                     tbl_dict[tst_name_mod]["history"][item[
 359                                         "title"]].append(tst_data["throughput"][
 360                                         "PDR"]["LOWER"])
 361                             elif table["include-tests"] == "NDR":
 362                                 if tst_data["type"] == "NDR":
 363                                     tbl_dict[tst_name_mod]["history"][
 364                                         item["title"]].\
 365                                         append(tst_data["throughput"]["value"])
 366                                 elif tst_data["type"] == "NDRPDR":
 367                                     tbl_dict[tst_name_mod]["history"][item[
 368                                         "title"]].append(tst_data["throughput"][
 369                                         "NDR"]["LOWER"])
 370                             else:
 371                                 continue
 372                         except (TypeError, KeyError):
 373                             pass
 374
 375     tbl_lst = list()
 376     for tst_name in tbl_dict.keys():
 377         item = [tbl_dict[tst_name]["name"], ]
 378         if history:
 379             if tbl_dict[tst_name].get("history", None) is not None:
 380                 for hist_data in tbl_dict[tst_name]["history"].values():
 381                     if hist_data:
 382                         item.append(round(mean(hist_data) / 1000000, 2))
 383                         item.append(round(stdev(hist_data) / 1000000, 2))
 384                     else:
 385                         item.extend([None, None])
 386             else:
 387                 item.extend([None, None])
 388         data_t = tbl_dict[tst_name]["ref-data"]
 389         if data_t:
 390             item.append(round(mean(data_t) / 1000000, 2))
 391             item.append(round(stdev(data_t) / 1000000, 2))
 392         else:
 393             item.extend([None, None])
 394         data_t = tbl_dict[tst_name]["cmp-data"]
 395         if data_t:
 396             item.append(round(mean(data_t) / 1000000, 2))
 397             item.append(round(stdev(data_t) / 1000000, 2))
 398         else:
 399             item.extend([None, None])
 400         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
 401             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 402         if len(item) == len(header):
 403             tbl_lst.append(item)
 404
 405     # Sort the table according to the relative change
 406     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 407
 408     # Generate csv tables:
 409     csv_file = "{0}.csv".format(table["output-file"])
 410     with open(csv_file, "w") as file_handler:
 411         file_handler.write(header_str)
 412         for test in tbl_lst:
 413             file_handler.write(",".join([str(item) for item in test]) + "\n")
 414
 415     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
 416
 417
 418 def table_nics_comparison(table, input_data):
 419     """Generate the table(s) with algorithm: table_nics_comparison
 420     specified in the specification file.
 421
 422     :param table: Table to generate.
 423     :param input_data: Data to process.
 424     :type table: pandas.Series
 425     :type input_data: InputData
 426     """
 427
 428     logging.info("  Generating the table {0} ...".
 429                  format(table.get("title", "")))
 430
 431     # Transform the data
 432     logging.info("    Creating the data set for the {0} '{1}'.".
 433                  format(table.get("type", ""), table.get("title", "")))
 434     data = input_data.filter_data(table, continue_on_error=True)
 435
 436     # Prepare the header of the tables
 437     try:
 438         header = ["Test case", ]
 439
 440         if table["include-tests"] == "MRR":
 441             hdr_param = "Receive Rate"
 442         else:
 443             hdr_param = "Throughput"
 444
 445         header.extend(
 446             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
 447              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 448              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
 449              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 450              "Delta [%]"])
 451         header_str = ",".join(header) + "\n"
 452     except (AttributeError, KeyError) as err:
 453         logging.error("The model is invalid, missing parameter: {0}".
 454                       format(err))
 455         return
 456
 457     # Prepare data to the table:
 458     tbl_dict = dict()
 459     for job, builds in table["data"].items():
 460         for build in builds:
 461             for tst_name, tst_data in data[job][str(build)].iteritems():
 462                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
 463                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
 464                     replace("-ndrdisc", "").replace("-pdr", "").\
 465                     replace("-ndr", "").\
 466                     replace("1t1c", "1c").replace("2t1c", "1c").\
 467                     replace("2t2c", "2c").replace("4t2c", "2c").\
 468                     replace("4t4c", "4c").replace("8t4c", "4c")
 469                 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
 470                 if tbl_dict.get(tst_name_mod, None) is None:
 471                     name = "-".join(tst_data["name"].split("-")[:-1])
 472                     tbl_dict[tst_name_mod] = {"name": name,
 473                                               "ref-data": list(),
 474                                               "cmp-data": list()}
 475                 try:
 476                     if table["include-tests"] == "MRR":
 477                         result = tst_data["result"]["receive-rate"].avg
 478                     elif table["include-tests"] == "PDR":
 479                         result = tst_data["throughput"]["PDR"]["LOWER"]
 480                     elif table["include-tests"] == "NDR":
 481                         result = tst_data["throughput"]["NDR"]["LOWER"]
 482                     else:
 483                         result = None
 484
 485                     if result:
 486                         if table["reference"]["nic"] in tst_data["tags"]:
 487                             tbl_dict[tst_name_mod]["ref-data"].append(result)
 488                         elif table["compare"]["nic"] in tst_data["tags"]:
 489                             tbl_dict[tst_name_mod]["cmp-data"].append(result)
 490                 except (TypeError, KeyError) as err:
 491                     logging.debug("No data for {0}".format(tst_name))
 492                     logging.debug(repr(err))
 493                     # No data in output.xml for this test
 494
 495     tbl_lst = list()
 496     for tst_name in tbl_dict.keys():
 497         item = [tbl_dict[tst_name]["name"], ]
 498         data_t = tbl_dict[tst_name]["ref-data"]
 499         if data_t:
 500             item.append(round(mean(data_t) / 1000000, 2))
 501             item.append(round(stdev(data_t) / 1000000, 2))
 502         else:
 503             item.extend([None, None])
 504         data_t = tbl_dict[tst_name]["cmp-data"]
 505         if data_t:
 506             item.append(round(mean(data_t) / 1000000, 2))
 507             item.append(round(stdev(data_t) / 1000000, 2))
 508         else:
 509             item.extend([None, None])
 510         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
 511             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 512         if len(item) == len(header):
 513             tbl_lst.append(item)
 514
 515     # Sort the table according to the relative change
 516     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 517
 518     # Generate csv tables:
 519     csv_file = "{0}.csv".format(table["output-file"])
 520     with open(csv_file, "w") as file_handler:
 521         file_handler.write(header_str)
 522         for test in tbl_lst:
 523             file_handler.write(",".join([str(item) for item in test]) + "\n")
 524
 525     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
 526
 527
 528 def table_soak_vs_ndr(table, input_data):
 529     """Generate the table(s) with algorithm: table_soak_vs_ndr
 530     specified in the specification file.
 531
 532     :param table: Table to generate.
 533     :param input_data: Data to process.
 534     :type table: pandas.Series
 535     :type input_data: InputData
 536     """
 537
 538     logging.info("  Generating the table {0} ...".
 539                  format(table.get("title", "")))
 540
 541     # Transform the data
 542     logging.info("    Creating the data set for the {0} '{1}'.".
 543                  format(table.get("type", ""), table.get("title", "")))
 544     data = input_data.filter_data(table, continue_on_error=True)
 545
 546     # Prepare the header of the table
 547     try:
 548         header = [
 549             "Test case",
 550             "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 551             "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 552             "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 553             "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 554             "Delta [%]", "Stdev of delta [%]"]
 555         header_str = ",".join(header) + "\n"
 556     except (AttributeError, KeyError) as err:
 557         logging.error("The model is invalid, missing parameter: {0}".
 558                       format(err))
 559         return
 560
 561     # Create a list of available SOAK test results:
 562     tbl_dict = dict()
 563     for job, builds in table["compare"]["data"].items():
 564         for build in builds:
 565             for tst_name, tst_data in data[job][str(build)].iteritems():
 566                 if tst_data["type"] == "SOAK":
 567                     tst_name_mod = tst_name.replace("-soak", "")
 568                     if tbl_dict.get(tst_name_mod, None) is None:
 569                         groups = re.search(REGEX_NIC, tst_data["parent"])
 570                         nic = groups.group(0) if groups else ""
 571                         name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
 572                                                               split("-")[:-1]))
 573                         tbl_dict[tst_name_mod] = {
 574                             "name": name,
 575                             "ref-data": list(),
 576                             "cmp-data": list()
 577                         }
 578                     try:
 579                         tbl_dict[tst_name_mod]["cmp-data"].append(
 580                             tst_data["throughput"]["LOWER"])
 581                     except (KeyError, TypeError):
 582                         pass
 583     tests_lst = tbl_dict.keys()
 584
 585     # Add corresponding NDR test results:
 586     for job, builds in table["reference"]["data"].items():
 587         for build in builds:
 588             for tst_name, tst_data in data[job][str(build)].iteritems():
 589                 tst_name_mod = tst_name.replace("-ndrpdr", "").\
 590                     replace("-mrr", "")
 591                 if tst_name_mod in tests_lst:
 592                     try:
 593                         if tst_data["type"] in ("NDRPDR", "MRR", "BMRR"):
 594                             if table["include-tests"] == "MRR":
 595                                 result = tst_data["result"]["receive-rate"].avg
 596                             elif table["include-tests"] == "PDR":
 597                                 result = tst_data["throughput"]["PDR"]["LOWER"]
 598                             elif table["include-tests"] == "NDR":
 599                                 result = tst_data["throughput"]["NDR"]["LOWER"]
 600                             else:
 601                                 result = None
 602                             if result is not None:
 603                                 tbl_dict[tst_name_mod]["ref-data"].append(
 604                                     result)
 605                     except (KeyError, TypeError):
 606                         continue
 607
 608     tbl_lst = list()
 609     for tst_name in tbl_dict.keys():
 610         item = [tbl_dict[tst_name]["name"], ]
 611         data_r = tbl_dict[tst_name]["ref-data"]
 612         if data_r:
 613             data_r_mean = mean(data_r)
 614             item.append(round(data_r_mean / 1000000, 2))
 615             data_r_stdev = stdev(data_r)
 616             item.append(round(data_r_stdev / 1000000, 2))
 617         else:
 618             data_r_mean = None
 619             data_r_stdev = None
 620             item.extend([None, None])
 621         data_c = tbl_dict[tst_name]["cmp-data"]
 622         if data_c:
 623             data_c_mean = mean(data_c)
 624             item.append(round(data_c_mean / 1000000, 2))
 625             data_c_stdev = stdev(data_c)
 626             item.append(round(data_c_stdev / 1000000, 2))
 627         else:
 628             data_c_mean = None
 629             data_c_stdev = None
 630             item.extend([None, None])
 631         if data_r_mean and data_c_mean:
 632             delta, d_stdev = relative_change_stdev(
 633                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
 634             item.append(round(delta, 2))
 635             item.append(round(d_stdev, 2))
 636             tbl_lst.append(item)
 637
 638     # Sort the table according to the relative change
 639     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 640
 641     # Generate csv tables:
 642     csv_file = "{0}.csv".format(table["output-file"])
 643     with open(csv_file, "w") as file_handler:
 644         file_handler.write(header_str)
 645         for test in tbl_lst:
 646             file_handler.write(",".join([str(item) for item in test]) + "\n")
 647
 648     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
 649
 650
 651 def table_performance_trending_dashboard(table, input_data):
 652     """Generate the table(s) with algorithm:
 653     table_performance_trending_dashboard
 654     specified in the specification file.
 655
 656     :param table: Table to generate.
 657     :param input_data: Data to process.
 658     :type table: pandas.Series
 659     :type input_data: InputData
 660     """
 661
 662     logging.info("  Generating the table {0} ...".
 663                  format(table.get("title", "")))
 664
 665     # Transform the data
 666     logging.info("    Creating the data set for the {0} '{1}'.".
 667                  format(table.get("type", ""), table.get("title", "")))
 668     data = input_data.filter_data(table, continue_on_error=True)
 669
 670     # Prepare the header of the tables
 671     header = ["Test Case",
 672               "Trend [Mpps]",
 673               "Short-Term Change [%]",
 674               "Long-Term Change [%]",
 675               "Regressions [#]",
 676               "Progressions [#]"
 677               ]
 678     header_str = ",".join(header) + "\n"
 679
 680     # Prepare data to the table:
 681     tbl_dict = dict()
 682     for job, builds in table["data"].items():
 683         for build in builds:
 684             for tst_name, tst_data in data[job][str(build)].iteritems():
 685                 if tst_name.lower() in table.get("ignore-list", list()):
 686                     continue
 687                 if tbl_dict.get(tst_name, None) is None:
 688                     groups = re.search(REGEX_NIC, tst_data["parent"])
 689                     if not groups:
 690                         continue
 691                     nic = groups.group(0)
 692                     tbl_dict[tst_name] = {
 693                         "name": "{0}-{1}".format(nic, tst_data["name"]),
 694                         "data": OrderedDict()}
 695                 try:
 696                     tbl_dict[tst_name]["data"][str(build)] = \
 697                         tst_data["result"]["receive-rate"]
 698                 except (TypeError, KeyError):
 699                     pass  # No data in output.xml for this test
 700
 701     tbl_lst = list()
 702     for tst_name in tbl_dict.keys():
 703         data_t = tbl_dict[tst_name]["data"]
 704         if len(data_t) < 2:
 705             continue
 706
 707         classification_lst, avgs = classify_anomalies(data_t)
 708
 709         win_size = min(len(data_t), table["window"])
 710         long_win_size = min(len(data_t), table["long-trend-window"])
 711
 712         try:
 713             max_long_avg = max(
 714                 [x for x in avgs[-long_win_size:-win_size]
 715                  if not isnan(x)])
 716         except ValueError:
 717             max_long_avg = nan
 718         last_avg = avgs[-1]
 719         avg_week_ago = avgs[max(-win_size, -len(avgs))]
 720
 721         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
 722             rel_change_last = nan
 723         else:
 724             rel_change_last = round(
 725                 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
 726
 727         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
 728             rel_change_long = nan
 729         else:
 730             rel_change_long = round(
 731                 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
 732
 733         if classification_lst:
 734             if isnan(rel_change_last) and isnan(rel_change_long):
 735                 continue
 736             if (isnan(last_avg) or
 737                 isnan(rel_change_last) or
 738                 isnan(rel_change_long)):
 739                 continue
 740             tbl_lst.append(
 741                 [tbl_dict[tst_name]["name"],
 742                  round(last_avg / 1000000, 2),
 743                  rel_change_last,
 744                  rel_change_long,
 745                  classification_lst[-win_size:].count("regression"),
 746                  classification_lst[-win_size:].count("progression")])
 747
 748     tbl_lst.sort(key=lambda rel: rel[0])
 749
 750     tbl_sorted = list()
 751     for nrr in range(table["window"], -1, -1):
 752         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
 753         for nrp in range(table["window"], -1, -1):
 754             tbl_out = [item for item in tbl_reg if item[5] == nrp]
 755             tbl_out.sort(key=lambda rel: rel[2])
 756             tbl_sorted.extend(tbl_out)
 757
 758     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
 759
 760     logging.info("    Writing file: '{0}'".format(file_name))
 761     with open(file_name, "w") as file_handler:
 762         file_handler.write(header_str)
 763         for test in tbl_sorted:
 764             file_handler.write(",".join([str(item) for item in test]) + '\n')
 765
 766     txt_file_name = "{0}.txt".format(table["output-file"])
 767     logging.info("    Writing file: '{0}'".format(txt_file_name))
 768     convert_csv_to_pretty_txt(file_name, txt_file_name)
 769
 770
 771 def _generate_url(base, testbed, test_name):
 772     """Generate URL to a trending plot from the name of the test case.
 773
 774     :param base: The base part of URL common to all test cases.
 775     :param testbed: The testbed used for testing.
 776     :param test_name: The name of the test case.
 777     :type base: str
 778     :type testbed: str
 779     :type test_name: str
 780     :returns: The URL to the plot with the trending data for the given test
 781         case.
 782     :rtype str
 783     """
 784
 785     url = base
 786     file_name = ""
 787     anchor = ".html#"
 788     feature = ""
 789
 790     if "lbdpdk" in test_name or "lbvpp" in test_name:
 791         file_name = "link_bonding"
 792
 793     elif "114b" in test_name and "vhost" in test_name:
 794         file_name = "vts"
 795
 796     elif "testpmd" in test_name or "l3fwd" in test_name:
 797         file_name = "dpdk"
 798
 799     elif "memif" in test_name:
 800         file_name = "container_memif"
 801         feature = "-base"
 802
 803     elif "srv6" in test_name:
 804         file_name = "srv6"
 805
 806     elif "vhost" in test_name:
 807         if "l2xcbase" in test_name or "l2bdbasemaclrn" in test_name:
 808             file_name = "vm_vhost_l2"
 809             if "114b" in test_name:
 810                 feature = ""
 811             elif "l2xcbase" in test_name and "x520" in test_name:
 812                 feature = "-base-l2xc"
 813             elif "l2bdbasemaclrn" in test_name and "x520" in test_name:
 814                 feature = "-base-l2bd"
 815             else:
 816                 feature = "-base"
 817         elif "ip4base" in test_name:
 818             file_name = "vm_vhost_ip4"
 819             feature = "-base"
 820
 821     elif "ipsecbasetnlsw" in test_name:
 822         file_name = "ipsecsw"
 823         feature = "-base-scale"
 824
 825     elif "ipsec" in test_name:
 826         file_name = "ipsec"
 827         feature = "-base-scale"
 828         if "hw-" in test_name:
 829             file_name = "ipsechw"
 830         elif "sw-" in test_name:
 831             file_name = "ipsecsw"
 832
 833     elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name:
 834         file_name = "ip4_tunnels"
 835         feature = "-base"
 836
 837     elif "ip4base" in test_name or "ip4scale" in test_name:
 838         file_name = "ip4"
 839         if "xl710" in test_name:
 840             feature = "-base-scale-features"
 841         elif "iacl" in test_name:
 842             feature = "-features-iacl"
 843         elif "oacl" in test_name:
 844             feature = "-features-oacl"
 845         elif "snat" in test_name or "cop" in test_name:
 846             feature = "-features"
 847         else:
 848             feature = "-base-scale"
 849
 850     elif "ip6base" in test_name or "ip6scale" in test_name:
 851         file_name = "ip6"
 852         feature = "-base-scale"
 853
 854     elif "l2xcbase" in test_name or "l2xcscale" in test_name \
 855             or "l2bdbasemaclrn" in test_name or "l2bdscale" in test_name \
 856             or "l2dbbasemaclrn" in test_name or "l2dbscale" in test_name:
 857         file_name = "l2"
 858         if "macip" in test_name:
 859             feature = "-features-macip"
 860         elif "iacl" in test_name:
 861             feature = "-features-iacl"
 862         elif "oacl" in test_name:
 863             feature = "-features-oacl"
 864         else:
 865             feature = "-base-scale"
 866
 867     if "x520" in test_name:
 868         nic = "x520-"
 869     elif "x710" in test_name:
 870         nic = "x710-"
 871     elif "xl710" in test_name:
 872         nic = "xl710-"
 873     elif "xxv710" in test_name:
 874         nic = "xxv710-"
 875     elif "vic1227" in test_name:
 876         nic = "vic1227-"
 877     elif "vic1385" in test_name:
 878         nic = "vic1385-"
 879     else:
 880         nic = ""
 881     anchor += nic
 882
 883     if "64b" in test_name:
 884         framesize = "64b"
 885     elif "78b" in test_name:
 886         framesize = "78b"
 887     elif "imix" in test_name:
 888         framesize = "imix"
 889     elif "9000b" in test_name:
 890         framesize = "9000b"
 891     elif "1518b" in test_name:
 892         framesize = "1518b"
 893     elif "114b" in test_name:
 894         framesize = "114b"
 895     else:
 896         framesize = ""
 897     anchor += framesize + '-'
 898
 899     if "1t1c" in test_name:
 900         anchor += "1t1c"
 901     elif "2t2c" in test_name:
 902         anchor += "2t2c"
 903     elif "4t4c" in test_name:
 904         anchor += "4t4c"
 905     elif "2t1c" in test_name:
 906         anchor += "2t1c"
 907     elif "4t2c" in test_name:
 908         anchor += "4t2c"
 909     elif "8t4c" in test_name:
 910         anchor += "8t4c"
 911
 912     return url + file_name + '-' + testbed + '-' + nic + framesize + \
 913         feature.replace("-int", "").replace("-tnl", "") + anchor + feature
 914
 915
 916 def table_performance_trending_dashboard_html(table, input_data):
 917     """Generate the table(s) with algorithm:
 918     table_performance_trending_dashboard_html specified in the specification
 919     file.
 920
 921     :param table: Table to generate.
 922     :param input_data: Data to process.
 923     :type table: dict
 924     :type input_data: InputData
 925     """
 926
 927     testbed = table.get("testbed", None)
 928     if testbed is None:
 929         logging.error("The testbed is not defined for the table '{0}'.".
 930                       format(table.get("title", "")))
 931         return
 932
 933     logging.info("  Generating the table {0} ...".
 934                  format(table.get("title", "")))
 935
 936     try:
 937         with open(table["input-file"], 'rb') as csv_file:
 938             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 939             csv_lst = [item for item in csv_content]
 940     except KeyError:
 941         logging.warning("The input file is not defined.")
 942         return
 943     except csv.Error as err:
 944         logging.warning("Not possible to process the file '{0}'.\n{1}".
 945                         format(table["input-file"], err))
 946         return
 947
 948     # Table:
 949     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
 950
 951     # Table header:
 952     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
 953     for idx, item in enumerate(csv_lst[0]):
 954         alignment = "left" if idx == 0 else "center"
 955         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
 956         th.text = item
 957
 958     # Rows:
 959     colors = {"regression": ("#ffcccc", "#ff9999"),
 960               "progression": ("#c6ecc6", "#9fdf9f"),
 961               "normal": ("#e9f1fb", "#d4e4f7")}
 962     for r_idx, row in enumerate(csv_lst[1:]):
 963         if int(row[4]):
 964             color = "regression"
 965         elif int(row[5]):
 966             color = "progression"
 967         else:
 968             color = "normal"
 969         background = colors[color][r_idx % 2]
 970         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
 971
 972         # Columns:
 973         for c_idx, item in enumerate(row):
 974             alignment = "left" if c_idx == 0 else "center"
 975             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
 976             # Name:
 977             if c_idx == 0:
 978                 url = _generate_url("../trending/", testbed, item)
 979                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
 980                 ref.text = item
 981             else:
 982                 td.text = item
 983     try:
 984         with open(table["output-file"], 'w') as html_file:
 985             logging.info("    Writing file: '{0}'".format(table["output-file"]))
 986             html_file.write(".. raw:: html\n\n\t")
 987             html_file.write(ET.tostring(dashboard))
 988             html_file.write("\n\t<p><br><br></p>\n")
 989     except KeyError:
 990         logging.warning("The output file is not defined.")
 991         return
 992
 993
 994 def table_last_failed_tests(table, input_data):
 995     """Generate the table(s) with algorithm: table_last_failed_tests
 996     specified in the specification file.
 997
 998     :param table: Table to generate.
 999     :param input_data: Data to process.
1000     :type table: pandas.Series
1001     :type input_data: InputData
1002     """
1003
1004     logging.info("  Generating the table {0} ...".
1005                  format(table.get("title", "")))
1006
1007     # Transform the data
1008     logging.info("    Creating the data set for the {0} '{1}'.".
1009                  format(table.get("type", ""), table.get("title", "")))
1010     data = input_data.filter_data(table, continue_on_error=True)
1011
1012     if data is None or data.empty:
1013         logging.warn("    No data for the {0} '{1}'.".
1014                      format(table.get("type", ""), table.get("title", "")))
1015         return
1016
1017     tbl_list = list()
1018     for job, builds in table["data"].items():
1019         for build in builds:
1020             build = str(build)
1021             try:
1022                 version = input_data.metadata(job, build).get("version", "")
1023             except KeyError:
1024                 logging.error("Data for {job}: {build} is not present.".
1025                               format(job=job, build=build))
1026                 return
1027             tbl_list.append(build)
1028             tbl_list.append(version)
1029             for tst_name, tst_data in data[job][build].iteritems():
1030                 if tst_data["status"] != "FAIL":
1031                     continue
1032                 groups = re.search(REGEX_NIC, tst_data["parent"])
1033                 if not groups:
1034                     continue
1035                 nic = groups.group(0)
1036                 tbl_list.append("{0}-{1}".format(nic, tst_data["name"]))
1037
1038     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1039     logging.info("    Writing file: '{0}'".format(file_name))
1040     with open(file_name, "w") as file_handler:
1041         for test in tbl_list:
1042             file_handler.write(test + '\n')
1043
1044
1045 def table_failed_tests(table, input_data):
1046     """Generate the table(s) with algorithm: table_failed_tests
1047     specified in the specification file.
1048
1049     :param table: Table to generate.
1050     :param input_data: Data to process.
1051     :type table: pandas.Series
1052     :type input_data: InputData
1053     """
1054
1055     logging.info("  Generating the table {0} ...".
1056                  format(table.get("title", "")))
1057
1058     # Transform the data
1059     logging.info("    Creating the data set for the {0} '{1}'.".
1060                  format(table.get("type", ""), table.get("title", "")))
1061     data = input_data.filter_data(table, continue_on_error=True)
1062
1063     # Prepare the header of the tables
1064     header = ["Test Case",
1065               "Failures [#]",
1066               "Last Failure [Time]",
1067               "Last Failure [VPP-Build-Id]",
1068               "Last Failure [CSIT-Job-Build-Id]"]
1069
1070     # Generate the data for the table according to the model in the table
1071     # specification
1072
1073     now = dt.utcnow()
1074     timeperiod = timedelta(int(table.get("window", 7)))
1075
1076     tbl_dict = dict()
1077     for job, builds in table["data"].items():
1078         for build in builds:
1079             build = str(build)
1080             for tst_name, tst_data in data[job][build].iteritems():
1081                 if tst_name.lower() in table.get("ignore-list", list()):
1082                     continue
1083                 if tbl_dict.get(tst_name, None) is None:
1084                     groups = re.search(REGEX_NIC, tst_data["parent"])
1085                     if not groups:
1086                         continue
1087                     nic = groups.group(0)
1088                     tbl_dict[tst_name] = {
1089                         "name": "{0}-{1}".format(nic, tst_data["name"]),
1090                         "data": OrderedDict()}
1091                 try:
1092                     generated = input_data.metadata(job, build).\
1093                         get("generated", "")
1094                     if not generated:
1095                         continue
1096                     then = dt.strptime(generated, "%Y%m%d %H:%M")
1097                     if (now - then) <= timeperiod:
1098                         tbl_dict[tst_name]["data"][build] = (
1099                             tst_data["status"],
1100                             generated,
1101                             input_data.metadata(job, build).get("version", ""),
1102                             build)
1103                 except (TypeError, KeyError) as err:
1104                     logging.warning("tst_name: {} - err: {}".
1105                                     format(tst_name, repr(err)))
1106
1107     max_fails = 0
1108     tbl_lst = list()
1109     for tst_data in tbl_dict.values():
1110         fails_nr = 0
1111         for val in tst_data["data"].values():
1112             if val[0] == "FAIL":
1113                 fails_nr += 1
1114                 fails_last_date = val[1]
1115                 fails_last_vpp = val[2]
1116                 fails_last_csit = val[3]
1117         if fails_nr:
1118             max_fails = fails_nr if fails_nr > max_fails else max_fails
1119             tbl_lst.append([tst_data["name"],
1120                             fails_nr,
1121                             fails_last_date,
1122                             fails_last_vpp,
1123                             "mrr-daily-build-{0}".format(fails_last_csit)])
1124
1125     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1126     tbl_sorted = list()
1127     for nrf in range(max_fails, -1, -1):
1128         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1129         tbl_sorted.extend(tbl_fails)
1130     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1131
1132     logging.info("    Writing file: '{0}'".format(file_name))
1133     with open(file_name, "w") as file_handler:
1134         file_handler.write(",".join(header) + "\n")
1135         for test in tbl_sorted:
1136             file_handler.write(",".join([str(item) for item in test]) + '\n')
1137
1138     txt_file_name = "{0}.txt".format(table["output-file"])
1139     logging.info("    Writing file: '{0}'".format(txt_file_name))
1140     convert_csv_to_pretty_txt(file_name, txt_file_name)
1141
1142
1143 def table_failed_tests_html(table, input_data):
1144     """Generate the table(s) with algorithm: table_failed_tests_html
1145     specified in the specification file.
1146
1147     :param table: Table to generate.
1148     :param input_data: Data to process.
1149     :type table: pandas.Series
1150     :type input_data: InputData
1151     """
1152
1153     testbed = table.get("testbed", None)
1154     if testbed is None:
1155         logging.error("The testbed is not defined for the table '{0}'.".
1156                       format(table.get("title", "")))
1157         return
1158
1159     logging.info("  Generating the table {0} ...".
1160                  format(table.get("title", "")))
1161
1162     try:
1163         with open(table["input-file"], 'rb') as csv_file:
1164             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
1165             csv_lst = [item for item in csv_content]
1166     except KeyError:
1167         logging.warning("The input file is not defined.")
1168         return
1169     except csv.Error as err:
1170         logging.warning("Not possible to process the file '{0}'.\n{1}".
1171                         format(table["input-file"], err))
1172         return
1173
1174     # Table:
1175     failed_tests = ET.Element("table", attrib=dict(width="100%", border='0'))
1176
1177     # Table header:
1178     tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor="#7eade7"))
1179     for idx, item in enumerate(csv_lst[0]):
1180         alignment = "left" if idx == 0 else "center"
1181         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
1182         th.text = item
1183
1184     # Rows:
1185     colors = ("#e9f1fb", "#d4e4f7")
1186     for r_idx, row in enumerate(csv_lst[1:]):
1187         background = colors[r_idx % 2]
1188         tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor=background))
1189
1190         # Columns:
1191         for c_idx, item in enumerate(row):
1192             alignment = "left" if c_idx == 0 else "center"
1193             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1194             # Name:
1195             if c_idx == 0:
1196                 url = _generate_url("../trending/", testbed, item)
1197                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1198                 ref.text = item
1199             else:
1200                 td.text = item
1201     try:
1202         with open(table["output-file"], 'w') as html_file:
1203             logging.info("    Writing file: '{0}'".format(table["output-file"]))
1204             html_file.write(".. raw:: html\n\n\t")
1205             html_file.write(ET.tostring(failed_tests))
1206             html_file.write("\n\t<p><br><br></p>\n")
1207     except KeyError:
1208         logging.warning("The output file is not defined.")
1209         return