resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from string import replace
  23 from collections import OrderedDict
  24 from numpy import nan, isnan
  25 from xml.etree import ElementTree as ET
  26 from datetime import datetime as dt
  27 from datetime import timedelta
  28
  29 from utils import mean, stdev, relative_change, classify_anomalies, \
  30     convert_csv_to_pretty_txt, relative_change_stdev
  31
  32
  33 REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*')
  34
  35
  36 def generate_tables(spec, data):
  37     """Generate all tables specified in the specification file.
  38
  39     :param spec: Specification read from the specification file.
  40     :param data: Data to process.
  41     :type spec: Specification
  42     :type data: InputData
  43     """
  44
  45     logging.info("Generating the tables ...")
  46     for table in spec.tables:
  47         try:
  48             eval(table["algorithm"])(table, data)
  49         except NameError as err:
  50             logging.error("Probably algorithm '{alg}' is not defined: {err}".
  51                           format(alg=table["algorithm"], err=repr(err)))
  52     logging.info("Done.")
  53
  54
  55 def table_details(table, input_data):
  56     """Generate the table(s) with algorithm: table_detailed_test_results
  57     specified in the specification file.
  58
  59     :param table: Table to generate.
  60     :param input_data: Data to process.
  61     :type table: pandas.Series
  62     :type input_data: InputData
  63     """
  64
  65     logging.info("  Generating the table {0} ...".
  66                  format(table.get("title", "")))
  67
  68     # Transform the data
  69     logging.info("    Creating the data set for the {0} '{1}'.".
  70                  format(table.get("type", ""), table.get("title", "")))
  71     data = input_data.filter_data(table)
  72
  73     # Prepare the header of the tables
  74     header = list()
  75     for column in table["columns"]:
  76         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  77
  78     # Generate the data for the table according to the model in the table
  79     # specification
  80     job = table["data"].keys()[0]
  81     build = str(table["data"][job][0])
  82     try:
  83         suites = input_data.suites(job, build)
  84     except KeyError:
  85         logging.error("    No data available. The table will not be generated.")
  86         return
  87
  88     for suite_longname, suite in suites.iteritems():
  89         # Generate data
  90         suite_name = suite["name"]
  91         table_lst = list()
  92         for test in data[job][build].keys():
  93             if data[job][build][test]["parent"] in suite_name:
  94                 row_lst = list()
  95                 for column in table["columns"]:
  96                     try:
  97                         col_data = str(data[job][build][test][column["data"].
  98                                        split(" ")[1]]).replace('"', '""')
  99                         if column["data"].split(" ")[1] in ("conf-history",
 100                                                             "show-run"):
 101                             col_data = replace(col_data, " |br| ", "",
 102                                                maxreplace=1)
 103                             col_data = " |prein| {0} |preout| ".\
 104                                 format(col_data[:-5])
 105                         row_lst.append('"{0}"'.format(col_data))
 106                     except KeyError:
 107                         row_lst.append("No data")
 108                 table_lst.append(row_lst)
 109
 110         # Write the data to file
 111         if table_lst:
 112             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 113                                             table["output-file-ext"])
 114             logging.info("      Writing file: '{}'".format(file_name))
 115             with open(file_name, "w") as file_handler:
 116                 file_handler.write(",".join(header) + "\n")
 117                 for item in table_lst:
 118                     file_handler.write(",".join(item) + "\n")
 119
 120     logging.info("  Done.")
 121
 122
 123 def table_merged_details(table, input_data):
 124     """Generate the table(s) with algorithm: table_merged_details
 125     specified in the specification file.
 126
 127     :param table: Table to generate.
 128     :param input_data: Data to process.
 129     :type table: pandas.Series
 130     :type input_data: InputData
 131     """
 132
 133     logging.info("  Generating the table {0} ...".
 134                  format(table.get("title", "")))
 135
 136     # Transform the data
 137     logging.info("    Creating the data set for the {0} '{1}'.".
 138                  format(table.get("type", ""), table.get("title", "")))
 139     data = input_data.filter_data(table)
 140     data = input_data.merge_data(data)
 141     data.sort_index(inplace=True)
 142
 143     logging.info("    Creating the data set for the {0} '{1}'.".
 144                  format(table.get("type", ""), table.get("title", "")))
 145     suites = input_data.filter_data(table, data_set="suites")
 146     suites = input_data.merge_data(suites)
 147
 148     # Prepare the header of the tables
 149     header = list()
 150     for column in table["columns"]:
 151         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 152
 153     for _, suite in suites.iteritems():
 154         # Generate data
 155         suite_name = suite["name"]
 156         table_lst = list()
 157         for test in data.keys():
 158             if data[test]["parent"] in suite_name:
 159                 row_lst = list()
 160                 for column in table["columns"]:
 161                     try:
 162                         col_data = str(data[test][column["data"].
 163                                        split(" ")[1]]).replace('"', '""')
 164                         col_data = replace(col_data, "No Data",
 165                                            "Not Captured     ")
 166                         if column["data"].split(" ")[1] in ("conf-history",
 167                                                             "show-run"):
 168                             col_data = replace(col_data, " |br| ", "",
 169                                                maxreplace=1)
 170                             col_data = " |prein| {0} |preout| ".\
 171                                 format(col_data[:-5])
 172                         row_lst.append('"{0}"'.format(col_data))
 173                     except KeyError:
 174                         row_lst.append('"Not captured"')
 175                 table_lst.append(row_lst)
 176
 177         # Write the data to file
 178         if table_lst:
 179             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 180                                             table["output-file-ext"])
 181             logging.info("      Writing file: '{}'".format(file_name))
 182             with open(file_name, "w") as file_handler:
 183                 file_handler.write(",".join(header) + "\n")
 184                 for item in table_lst:
 185                     file_handler.write(",".join(item) + "\n")
 186
 187     logging.info("  Done.")
 188
 189
 190 def table_performance_comparison(table, input_data):
 191     """Generate the table(s) with algorithm: table_performance_comparison
 192     specified in the specification file.
 193
 194     :param table: Table to generate.
 195     :param input_data: Data to process.
 196     :type table: pandas.Series
 197     :type input_data: InputData
 198     """
 199
 200     logging.info("  Generating the table {0} ...".
 201                  format(table.get("title", "")))
 202
 203     # Transform the data
 204     logging.info("    Creating the data set for the {0} '{1}'.".
 205                  format(table.get("type", ""), table.get("title", "")))
 206     data = input_data.filter_data(table, continue_on_error=True)
 207
 208     # Prepare the header of the tables
 209     try:
 210         header = ["Test case", ]
 211
 212         if table["include-tests"] == "MRR":
 213             hdr_param = "Rec Rate"
 214         else:
 215             hdr_param = "Thput"
 216
 217         history = table.get("history", None)
 218         if history:
 219             for item in history:
 220                 header.extend(
 221                     ["{0} {1} [Mpps]".format(item["title"], hdr_param),
 222                      "{0} Stdev [Mpps]".format(item["title"])])
 223         header.extend(
 224             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
 225              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 226              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
 227              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 228              "Delta [%]"])
 229         header_str = ",".join(header) + "\n"
 230     except (AttributeError, KeyError) as err:
 231         logging.error("The model is invalid, missing parameter: {0}".
 232                       format(err))
 233         return
 234
 235     # Prepare data to the table:
 236     tbl_dict = dict()
 237     for job, builds in table["reference"]["data"].items():
 238         topo = "2n-skx" if "2n-skx" in job else ""
 239         for build in builds:
 240             for tst_name, tst_data in data[job][str(build)].iteritems():
 241                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
 242                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
 243                     replace("-ndrdisc", "").replace("-pdr", "").\
 244                     replace("-ndr", "").\
 245                     replace("1t1c", "1c").replace("2t1c", "1c").\
 246                     replace("2t2c", "2c").replace("4t2c", "2c").\
 247                     replace("4t4c", "4c").replace("8t4c", "4c")
 248                 if "across topologies" in table["title"].lower():
 249                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
 250                 if tbl_dict.get(tst_name_mod, None) is None:
 251                     groups = re.search(REGEX_NIC, tst_data["parent"])
 252                     nic = groups.group(0) if groups else ""
 253                     name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
 254                                                           split("-")[:-1]))
 255                     if "across testbeds" in table["title"].lower() or \
 256                             "across topologies" in table["title"].lower():
 257                         name = name.\
 258                             replace("1t1c", "1c").replace("2t1c", "1c").\
 259                             replace("2t2c", "2c").replace("4t2c", "2c").\
 260                             replace("4t4c", "4c").replace("8t4c", "4c")
 261                     tbl_dict[tst_name_mod] = {"name": name,
 262                                               "ref-data": list(),
 263                                               "cmp-data": list()}
 264                 try:
 265                     # TODO: Re-work when NDRPDRDISC tests are not used
 266                     if table["include-tests"] == "MRR":
 267                         tbl_dict[tst_name_mod]["ref-data"]. \
 268                             append(tst_data["result"]["receive-rate"].avg)
 269                     elif table["include-tests"] == "PDR":
 270                         if tst_data["type"] == "PDR":
 271                             tbl_dict[tst_name_mod]["ref-data"]. \
 272                                 append(tst_data["throughput"]["value"])
 273                         elif tst_data["type"] == "NDRPDR":
 274                             tbl_dict[tst_name_mod]["ref-data"].append(
 275                                 tst_data["throughput"]["PDR"]["LOWER"])
 276                     elif table["include-tests"] == "NDR":
 277                         if tst_data["type"] == "NDR":
 278                             tbl_dict[tst_name_mod]["ref-data"]. \
 279                                 append(tst_data["throughput"]["value"])
 280                         elif tst_data["type"] == "NDRPDR":
 281                             tbl_dict[tst_name_mod]["ref-data"].append(
 282                                 tst_data["throughput"]["NDR"]["LOWER"])
 283                     else:
 284                         continue
 285                 except TypeError:
 286                     pass  # No data in output.xml for this test
 287
 288     for job, builds in table["compare"]["data"].items():
 289         for build in builds:
 290             for tst_name, tst_data in data[job][str(build)].iteritems():
 291                 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
 292                     replace("-ndrpdr", "").replace("-pdrdisc", ""). \
 293                     replace("-ndrdisc", "").replace("-pdr", ""). \
 294                     replace("-ndr", "").\
 295                     replace("1t1c", "1c").replace("2t1c", "1c").\
 296                     replace("2t2c", "2c").replace("4t2c", "2c").\
 297                     replace("4t4c", "4c").replace("8t4c", "4c")
 298                 if "across topologies" in table["title"].lower():
 299                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
 300                 if tbl_dict.get(tst_name_mod, None) is None:
 301                     groups = re.search(REGEX_NIC, tst_data["parent"])
 302                     nic = groups.group(0) if groups else ""
 303                     name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
 304                                                           split("-")[:-1]))
 305                     if "across testbeds" in table["title"].lower() or \
 306                             "across topologies" in table["title"].lower():
 307                         name = name.\
 308                             replace("1t1c", "1c").replace("2t1c", "1c").\
 309                             replace("2t2c", "2c").replace("4t2c", "2c").\
 310                             replace("4t4c", "4c").replace("8t4c", "4c")
 311                     tbl_dict[tst_name_mod] = {"name": name,
 312                                               "ref-data": list(),
 313                                               "cmp-data": list()}
 314                 try:
 315                     # TODO: Re-work when NDRPDRDISC tests are not used
 316                     if table["include-tests"] == "MRR":
 317                         tbl_dict[tst_name_mod]["cmp-data"]. \
 318                             append(tst_data["result"]["receive-rate"].avg)
 319                     elif table["include-tests"] == "PDR":
 320                         if tst_data["type"] == "PDR":
 321                             tbl_dict[tst_name_mod]["cmp-data"]. \
 322                                 append(tst_data["throughput"]["value"])
 323                         elif tst_data["type"] == "NDRPDR":
 324                             tbl_dict[tst_name_mod]["cmp-data"].append(
 325                                 tst_data["throughput"]["PDR"]["LOWER"])
 326                     elif table["include-tests"] == "NDR":
 327                         if tst_data["type"] == "NDR":
 328                             tbl_dict[tst_name_mod]["cmp-data"]. \
 329                                 append(tst_data["throughput"]["value"])
 330                         elif tst_data["type"] == "NDRPDR":
 331                             tbl_dict[tst_name_mod]["cmp-data"].append(
 332                                 tst_data["throughput"]["NDR"]["LOWER"])
 333                     else:
 334                         continue
 335                 except (KeyError, TypeError):
 336                     pass
 337     if history:
 338         for item in history:
 339             for job, builds in item["data"].items():
 340                 for build in builds:
 341                     for tst_name, tst_data in data[job][str(build)].iteritems():
 342                         tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
 343                             replace("-ndrpdr", "").replace("-pdrdisc", ""). \
 344                             replace("-ndrdisc", "").replace("-pdr", ""). \
 345                             replace("-ndr", "").\
 346                             replace("1t1c", "1c").replace("2t1c", "1c").\
 347                             replace("2t2c", "2c").replace("4t2c", "2c").\
 348                             replace("4t4c", "4c").replace("8t4c", "4c")
 349                         if "across topologies" in table["title"].lower():
 350                             tst_name_mod = tst_name_mod.replace("2n1l-", "")
 351                         if tbl_dict.get(tst_name_mod, None) is None:
 352                             continue
 353                         if tbl_dict[tst_name_mod].get("history", None) is None:
 354                             tbl_dict[tst_name_mod]["history"] = OrderedDict()
 355                         if tbl_dict[tst_name_mod]["history"].get(item["title"],
 356                                                              None) is None:
 357                             tbl_dict[tst_name_mod]["history"][item["title"]] = \
 358                                 list()
 359                         try:
 360                             # TODO: Re-work when NDRPDRDISC tests are not used
 361                             if table["include-tests"] == "MRR":
 362                                 tbl_dict[tst_name_mod]["history"][item["title"
 363                                 ]].append(tst_data["result"]["receive-rate"].
 364                                           avg)
 365                             elif table["include-tests"] == "PDR":
 366                                 if tst_data["type"] == "PDR":
 367                                     tbl_dict[tst_name_mod]["history"][
 368                                         item["title"]].\
 369                                         append(tst_data["throughput"]["value"])
 370                                 elif tst_data["type"] == "NDRPDR":
 371                                     tbl_dict[tst_name_mod]["history"][item[
 372                                         "title"]].append(tst_data["throughput"][
 373                                         "PDR"]["LOWER"])
 374                             elif table["include-tests"] == "NDR":
 375                                 if tst_data["type"] == "NDR":
 376                                     tbl_dict[tst_name_mod]["history"][
 377                                         item["title"]].\
 378                                         append(tst_data["throughput"]["value"])
 379                                 elif tst_data["type"] == "NDRPDR":
 380                                     tbl_dict[tst_name_mod]["history"][item[
 381                                         "title"]].append(tst_data["throughput"][
 382                                         "NDR"]["LOWER"])
 383                             else:
 384                                 continue
 385                         except (TypeError, KeyError):
 386                             pass
 387
 388     tbl_lst = list()
 389     footnote = False
 390     for tst_name in tbl_dict.keys():
 391         item = [tbl_dict[tst_name]["name"], ]
 392         if history:
 393             if tbl_dict[tst_name].get("history", None) is not None:
 394                 for hist_data in tbl_dict[tst_name]["history"].values():
 395                     if hist_data:
 396                         item.append(round(mean(hist_data) / 1000000, 2))
 397                         item.append(round(stdev(hist_data) / 1000000, 2))
 398                     else:
 399                         item.extend(["Not tested", "Not tested"])
 400             else:
 401                 item.extend(["Not tested", "Not tested"])
 402         data_t = tbl_dict[tst_name]["ref-data"]
 403         if data_t:
 404             item.append(round(mean(data_t) / 1000000, 2))
 405             item.append(round(stdev(data_t) / 1000000, 2))
 406         else:
 407             item.extend(["Not tested", "Not tested"])
 408         data_t = tbl_dict[tst_name]["cmp-data"]
 409         if data_t:
 410             item.append(round(mean(data_t) / 1000000, 2))
 411             item.append(round(stdev(data_t) / 1000000, 2))
 412         else:
 413             item.extend(["Not tested", "Not tested"])
 414         if item[-2] == "Not tested":
 415             pass
 416         elif item[-4] == "Not tested":
 417             item.append("New in CSIT-1908")
 418         elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]:
 419             item.append("See footnote [1]")
 420             footnote = True
 421         elif item[-4] != 0:
 422             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 423         if (len(item) == len(header)) and (item[-3] != "Not tested"):
 424             tbl_lst.append(item)
 425
 426     # Sort the table:
 427     # 1. New in CSIT-XXXX
 428     # 2. See footnote
 429     # 3. Delta
 430     tbl_new = list()
 431     tbl_see = list()
 432     tbl_delta = list()
 433     for item in tbl_lst:
 434         if isinstance(item[-1], str):
 435             if "New in CSIT" in item[-1]:
 436                 tbl_new.append(item)
 437             elif "See footnote" in item[-1]:
 438                 tbl_see.append(item)
 439         else:
 440             tbl_delta.append(item)
 441
 442     # Sort the tables:
 443     tbl_new.sort(key=lambda rel: rel[0], reverse=False)
 444     tbl_see.sort(key=lambda rel: rel[0], reverse=False)
 445     tbl_see.sort(key=lambda rel: rel[-1], reverse=False)
 446     tbl_delta.sort(key=lambda rel: rel[-1], reverse=True)
 447
 448     # Put the tables together:
 449     tbl_lst = list()
 450     tbl_lst.extend(tbl_new)
 451     tbl_lst.extend(tbl_see)
 452     tbl_lst.extend(tbl_delta)
 453
 454     # Generate csv tables:
 455     csv_file = "{0}.csv".format(table["output-file"])
 456     with open(csv_file, "w") as file_handler:
 457         file_handler.write(header_str)
 458         for test in tbl_lst:
 459             file_handler.write(",".join([str(item) for item in test]) + "\n")
 460
 461     txt_file_name = "{0}.txt".format(table["output-file"])
 462     convert_csv_to_pretty_txt(csv_file, txt_file_name)
 463
 464     if footnote:
 465         with open(txt_file_name, 'a') as txt_file:
 466             txt_file.writelines([
 467                 "\nFootnotes:\n",
 468                 "[1] CSIT-1908 changed test methodology of dot1q tests in "
 469                 "2-node testbeds, dot1q encapsulation is now used on both "
 470                 "links of SUT.\n",
 471                 "    Previously dot1q was used only on a single link with the "
 472                 "other link carrying untagged Ethernet frames. This changes "
 473                 "results\n",
 474                 "    in slightly lower throughput in CSIT-1908 for these "
 475                 "tests. See release notes."
 476             ])
 477
 478
 479 def table_performance_comparison_nic(table, input_data):
 480     """Generate the table(s) with algorithm: table_performance_comparison
 481     specified in the specification file.
 482
 483     :param table: Table to generate.
 484     :param input_data: Data to process.
 485     :type table: pandas.Series
 486     :type input_data: InputData
 487     """
 488
 489     logging.info("  Generating the table {0} ...".
 490                  format(table.get("title", "")))
 491
 492     # Transform the data
 493     logging.info("    Creating the data set for the {0} '{1}'.".
 494                  format(table.get("type", ""), table.get("title", "")))
 495     data = input_data.filter_data(table, continue_on_error=True)
 496
 497     # Prepare the header of the tables
 498     try:
 499         header = ["Test case", ]
 500
 501         if table["include-tests"] == "MRR":
 502             hdr_param = "Rec Rate"
 503         else:
 504             hdr_param = "Thput"
 505
 506         history = table.get("history", None)
 507         if history:
 508             for item in history:
 509                 header.extend(
 510                     ["{0} {1} [Mpps]".format(item["title"], hdr_param),
 511                      "{0} Stdev [Mpps]".format(item["title"])])
 512         header.extend(
 513             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
 514              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 515              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
 516              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 517              "Delta [%]"])
 518         header_str = ",".join(header) + "\n"
 519     except (AttributeError, KeyError) as err:
 520         logging.error("The model is invalid, missing parameter: {0}".
 521                       format(err))
 522         return
 523
 524     # Prepare data to the table:
 525     tbl_dict = dict()
 526     for job, builds in table["reference"]["data"].items():
 527         topo = "2n-skx" if "2n-skx" in job else ""
 528         for build in builds:
 529             for tst_name, tst_data in data[job][str(build)].iteritems():
 530                 if table["reference"]["nic"] not in tst_data["tags"]:
 531                     continue
 532                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
 533                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
 534                     replace("-ndrdisc", "").replace("-pdr", "").\
 535                     replace("-ndr", "").\
 536                     replace("1t1c", "1c").replace("2t1c", "1c").\
 537                     replace("2t2c", "2c").replace("4t2c", "2c").\
 538                     replace("4t4c", "4c").replace("8t4c", "4c")
 539                 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
 540                 if "across topologies" in table["title"].lower():
 541                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
 542                 if tbl_dict.get(tst_name_mod, None) is None:
 543                     name = "{0}".format("-".join(tst_data["name"].
 544                                                  split("-")[:-1]))
 545                     if "across testbeds" in table["title"].lower() or \
 546                             "across topologies" in table["title"].lower():
 547                         name = name.\
 548                             replace("1t1c", "1c").replace("2t1c", "1c").\
 549                             replace("2t2c", "2c").replace("4t2c", "2c").\
 550                             replace("4t4c", "4c").replace("8t4c", "4c")
 551                     tbl_dict[tst_name_mod] = {"name": name,
 552                                               "ref-data": list(),
 553                                               "cmp-data": list()}
 554                 try:
 555                     # TODO: Re-work when NDRPDRDISC tests are not used
 556                     if table["include-tests"] == "MRR":
 557                         tbl_dict[tst_name_mod]["ref-data"]. \
 558                             append(tst_data["result"]["receive-rate"].avg)
 559                     elif table["include-tests"] == "PDR":
 560                         if tst_data["type"] == "PDR":
 561                             tbl_dict[tst_name_mod]["ref-data"]. \
 562                                 append(tst_data["throughput"]["value"])
 563                         elif tst_data["type"] == "NDRPDR":
 564                             tbl_dict[tst_name_mod]["ref-data"].append(
 565                                 tst_data["throughput"]["PDR"]["LOWER"])
 566                     elif table["include-tests"] == "NDR":
 567                         if tst_data["type"] == "NDR":
 568                             tbl_dict[tst_name_mod]["ref-data"]. \
 569                                 append(tst_data["throughput"]["value"])
 570                         elif tst_data["type"] == "NDRPDR":
 571                             tbl_dict[tst_name_mod]["ref-data"].append(
 572                                 tst_data["throughput"]["NDR"]["LOWER"])
 573                     else:
 574                         continue
 575                 except TypeError:
 576                     pass  # No data in output.xml for this test
 577
 578     for job, builds in table["compare"]["data"].items():
 579         for build in builds:
 580             for tst_name, tst_data in data[job][str(build)].iteritems():
 581                 if table["compare"]["nic"] not in tst_data["tags"]:
 582                     continue
 583                 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
 584                     replace("-ndrpdr", "").replace("-pdrdisc", ""). \
 585                     replace("-ndrdisc", "").replace("-pdr", ""). \
 586                     replace("-ndr", "").\
 587                     replace("1t1c", "1c").replace("2t1c", "1c").\
 588                     replace("2t2c", "2c").replace("4t2c", "2c").\
 589                     replace("4t4c", "4c").replace("8t4c", "4c")
 590                 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
 591                 if "across topologies" in table["title"].lower():
 592                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
 593                 if tbl_dict.get(tst_name_mod, None) is None:
 594                     name = "{0}".format("-".join(tst_data["name"].
 595                                                  split("-")[:-1]))
 596                     if "across testbeds" in table["title"].lower() or \
 597                             "across topologies" in table["title"].lower():
 598                         name = name.\
 599                             replace("1t1c", "1c").replace("2t1c", "1c").\
 600                             replace("2t2c", "2c").replace("4t2c", "2c").\
 601                             replace("4t4c", "4c").replace("8t4c", "4c")
 602                     tbl_dict[tst_name_mod] = {"name": name,
 603                                               "ref-data": list(),
 604                                               "cmp-data": list()}
 605                 try:
 606                     # TODO: Re-work when NDRPDRDISC tests are not used
 607                     if table["include-tests"] == "MRR":
 608                         tbl_dict[tst_name_mod]["cmp-data"]. \
 609                             append(tst_data["result"]["receive-rate"].avg)
 610                     elif table["include-tests"] == "PDR":
 611                         if tst_data["type"] == "PDR":
 612                             tbl_dict[tst_name_mod]["cmp-data"]. \
 613                                 append(tst_data["throughput"]["value"])
 614                         elif tst_data["type"] == "NDRPDR":
 615                             tbl_dict[tst_name_mod]["cmp-data"].append(
 616                                 tst_data["throughput"]["PDR"]["LOWER"])
 617                     elif table["include-tests"] == "NDR":
 618                         if tst_data["type"] == "NDR":
 619                             tbl_dict[tst_name_mod]["cmp-data"]. \
 620                                 append(tst_data["throughput"]["value"])
 621                         elif tst_data["type"] == "NDRPDR":
 622                             tbl_dict[tst_name_mod]["cmp-data"].append(
 623                                 tst_data["throughput"]["NDR"]["LOWER"])
 624                     else:
 625                         continue
 626                 except (KeyError, TypeError):
 627                     pass
 628
 629     if history:
 630         for item in history:
 631             for job, builds in item["data"].items():
 632                 for build in builds:
 633                     for tst_name, tst_data in data[job][str(build)].iteritems():
 634                         if item["nic"] not in tst_data["tags"]:
 635                             continue
 636                         tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
 637                             replace("-ndrpdr", "").replace("-pdrdisc", ""). \
 638                             replace("-ndrdisc", "").replace("-pdr", ""). \
 639                             replace("-ndr", "").\
 640                             replace("1t1c", "1c").replace("2t1c", "1c").\
 641                             replace("2t2c", "2c").replace("4t2c", "2c").\
 642                             replace("4t4c", "4c").replace("8t4c", "4c")
 643                         tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
 644                         if "across topologies" in table["title"].lower():
 645                             tst_name_mod = tst_name_mod.replace("2n1l-", "")
 646                         if tbl_dict.get(tst_name_mod, None) is None:
 647                             continue
 648                         if tbl_dict[tst_name_mod].get("history", None) is None:
 649                             tbl_dict[tst_name_mod]["history"] = OrderedDict()
 650                         if tbl_dict[tst_name_mod]["history"].get(item["title"],
 651                                                              None) is None:
 652                             tbl_dict[tst_name_mod]["history"][item["title"]] = \
 653                                 list()
 654                         try:
 655                             # TODO: Re-work when NDRPDRDISC tests are not used
 656                             if table["include-tests"] == "MRR":
 657                                 tbl_dict[tst_name_mod]["history"][item["title"
 658                                 ]].append(tst_data["result"]["receive-rate"].
 659                                           avg)
 660                             elif table["include-tests"] == "PDR":
 661                                 if tst_data["type"] == "PDR":
 662                                     tbl_dict[tst_name_mod]["history"][
 663                                         item["title"]].\
 664                                         append(tst_data["throughput"]["value"])
 665                                 elif tst_data["type"] == "NDRPDR":
 666                                     tbl_dict[tst_name_mod]["history"][item[
 667                                         "title"]].append(tst_data["throughput"][
 668                                         "PDR"]["LOWER"])
 669                             elif table["include-tests"] == "NDR":
 670                                 if tst_data["type"] == "NDR":
 671                                     tbl_dict[tst_name_mod]["history"][
 672                                         item["title"]].\
 673                                         append(tst_data["throughput"]["value"])
 674                                 elif tst_data["type"] == "NDRPDR":
 675                                     tbl_dict[tst_name_mod]["history"][item[
 676                                         "title"]].append(tst_data["throughput"][
 677                                         "NDR"]["LOWER"])
 678                             else:
 679                                 continue
 680                         except (TypeError, KeyError):
 681                             pass
 682
 683     tbl_lst = list()
 684     footnote = False
 685     for tst_name in tbl_dict.keys():
 686         item = [tbl_dict[tst_name]["name"], ]
 687         if history:
 688             if tbl_dict[tst_name].get("history", None) is not None:
 689                 for hist_data in tbl_dict[tst_name]["history"].values():
 690                     if hist_data:
 691                         item.append(round(mean(hist_data) / 1000000, 2))
 692                         item.append(round(stdev(hist_data) / 1000000, 2))
 693                     else:
 694                         item.extend(["Not tested", "Not tested"])
 695             else:
 696                 item.extend(["Not tested", "Not tested"])
 697         data_t = tbl_dict[tst_name]["ref-data"]
 698         if data_t:
 699             item.append(round(mean(data_t) / 1000000, 2))
 700             item.append(round(stdev(data_t) / 1000000, 2))
 701         else:
 702             item.extend(["Not tested", "Not tested"])
 703         data_t = tbl_dict[tst_name]["cmp-data"]
 704         if data_t:
 705             item.append(round(mean(data_t) / 1000000, 2))
 706             item.append(round(stdev(data_t) / 1000000, 2))
 707         else:
 708             item.extend(["Not tested", "Not tested"])
 709         if item[-2] == "Not tested":
 710             pass
 711         elif item[-4] == "Not tested":
 712             item.append("New in CSIT-1908")
 713         elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]:
 714             item.append("See footnote [1]")
 715             footnote = True
 716         elif item[-4] != 0:
 717             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 718         if (len(item) == len(header)) and (item[-3] != "Not tested"):
 719             tbl_lst.append(item)
 720
 721     # Sort the table:
 722     # 1. New in CSIT-XXXX
 723     # 2. See footnote
 724     # 3. Delta
 725     tbl_new = list()
 726     tbl_see = list()
 727     tbl_delta = list()
 728     for item in tbl_lst:
 729         if isinstance(item[-1], str):
 730             if "New in CSIT" in item[-1]:
 731                 tbl_new.append(item)
 732             elif "See footnote" in item[-1]:
 733                 tbl_see.append(item)
 734         else:
 735             tbl_delta.append(item)
 736
 737     # Sort the tables:
 738     tbl_new.sort(key=lambda rel: rel[0], reverse=False)
 739     tbl_see.sort(key=lambda rel: rel[0], reverse=False)
 740     tbl_see.sort(key=lambda rel: rel[-1], reverse=False)
 741     tbl_delta.sort(key=lambda rel: rel[-1], reverse=True)
 742
 743     # Put the tables together:
 744     tbl_lst = list()
 745     tbl_lst.extend(tbl_new)
 746     tbl_lst.extend(tbl_see)
 747     tbl_lst.extend(tbl_delta)
 748
 749     # Generate csv tables:
 750     csv_file = "{0}.csv".format(table["output-file"])
 751     with open(csv_file, "w") as file_handler:
 752         file_handler.write(header_str)
 753         for test in tbl_lst:
 754             file_handler.write(",".join([str(item) for item in test]) + "\n")
 755
 756     txt_file_name = "{0}.txt".format(table["output-file"])
 757     convert_csv_to_pretty_txt(csv_file, txt_file_name)
 758
 759     if footnote:
 760         with open(txt_file_name, 'a') as txt_file:
 761             txt_file.writelines([
 762                 "\nFootnotes:\n",
 763                 "[1] CSIT-1908 changed test methodology of dot1q tests in "
 764                 "2-node testbeds, dot1q encapsulation is now used on both "
 765                 "links of SUT.\n",
 766                 "    Previously dot1q was used only on a single link with the "
 767                 "other link carrying untagged Ethernet frames. This changes "
 768                 "results\n",
 769                 "    in slightly lower throughput in CSIT-1908 for these "
 770                 "tests. See release notes."
 771             ])
 772
 773
 774 def table_nics_comparison(table, input_data):
 775     """Generate the table(s) with algorithm: table_nics_comparison
 776     specified in the specification file.
 777
 778     :param table: Table to generate.
 779     :param input_data: Data to process.
 780     :type table: pandas.Series
 781     :type input_data: InputData
 782     """
 783
 784     logging.info("  Generating the table {0} ...".
 785                  format(table.get("title", "")))
 786
 787     # Transform the data
 788     logging.info("    Creating the data set for the {0} '{1}'.".
 789                  format(table.get("type", ""), table.get("title", "")))
 790     data = input_data.filter_data(table, continue_on_error=True)
 791
 792     # Prepare the header of the tables
 793     try:
 794         header = ["Test case", ]
 795
 796         if table["include-tests"] == "MRR":
 797             hdr_param = "Rec Rate"
 798         else:
 799             hdr_param = "Thput"
 800
 801         header.extend(
 802             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
 803              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 804              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
 805              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 806              "Delta [%]"])
 807         header_str = ",".join(header) + "\n"
 808     except (AttributeError, KeyError) as err:
 809         logging.error("The model is invalid, missing parameter: {0}".
 810                       format(err))
 811         return
 812
 813     # Prepare data to the table:
 814     tbl_dict = dict()
 815     for job, builds in table["data"].items():
 816         for build in builds:
 817             for tst_name, tst_data in data[job][str(build)].iteritems():
 818                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
 819                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
 820                     replace("-ndrdisc", "").replace("-pdr", "").\
 821                     replace("-ndr", "").\
 822                     replace("1t1c", "1c").replace("2t1c", "1c").\
 823                     replace("2t2c", "2c").replace("4t2c", "2c").\
 824                     replace("4t4c", "4c").replace("8t4c", "4c")
 825                 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
 826                 if tbl_dict.get(tst_name_mod, None) is None:
 827                     name = "-".join(tst_data["name"].split("-")[:-1])
 828                     tbl_dict[tst_name_mod] = {"name": name,
 829                                               "ref-data": list(),
 830                                               "cmp-data": list()}
 831                 try:
 832                     if table["include-tests"] == "MRR":
 833                         result = tst_data["result"]["receive-rate"].avg
 834                     elif table["include-tests"] == "PDR":
 835                         result = tst_data["throughput"]["PDR"]["LOWER"]
 836                     elif table["include-tests"] == "NDR":
 837                         result = tst_data["throughput"]["NDR"]["LOWER"]
 838                     else:
 839                         result = None
 840
 841                     if result:
 842                         if table["reference"]["nic"] in tst_data["tags"]:
 843                             tbl_dict[tst_name_mod]["ref-data"].append(result)
 844                         elif table["compare"]["nic"] in tst_data["tags"]:
 845                             tbl_dict[tst_name_mod]["cmp-data"].append(result)
 846                 except (TypeError, KeyError) as err:
 847                     logging.debug("No data for {0}".format(tst_name))
 848                     logging.debug(repr(err))
 849                     # No data in output.xml for this test
 850
 851     tbl_lst = list()
 852     for tst_name in tbl_dict.keys():
 853         item = [tbl_dict[tst_name]["name"], ]
 854         data_t = tbl_dict[tst_name]["ref-data"]
 855         if data_t:
 856             item.append(round(mean(data_t) / 1000000, 2))
 857             item.append(round(stdev(data_t) / 1000000, 2))
 858         else:
 859             item.extend([None, None])
 860         data_t = tbl_dict[tst_name]["cmp-data"]
 861         if data_t:
 862             item.append(round(mean(data_t) / 1000000, 2))
 863             item.append(round(stdev(data_t) / 1000000, 2))
 864         else:
 865             item.extend([None, None])
 866         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
 867             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 868         if len(item) == len(header):
 869             tbl_lst.append(item)
 870
 871     # Sort the table according to the relative change
 872     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 873
 874     # Generate csv tables:
 875     csv_file = "{0}.csv".format(table["output-file"])
 876     with open(csv_file, "w") as file_handler:
 877         file_handler.write(header_str)
 878         for test in tbl_lst:
 879             file_handler.write(",".join([str(item) for item in test]) + "\n")
 880
 881     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
 882
 883
 884 def table_soak_vs_ndr(table, input_data):
 885     """Generate the table(s) with algorithm: table_soak_vs_ndr
 886     specified in the specification file.
 887
 888     :param table: Table to generate.
 889     :param input_data: Data to process.
 890     :type table: pandas.Series
 891     :type input_data: InputData
 892     """
 893
 894     logging.info("  Generating the table {0} ...".
 895                  format(table.get("title", "")))
 896
 897     # Transform the data
 898     logging.info("    Creating the data set for the {0} '{1}'.".
 899                  format(table.get("type", ""), table.get("title", "")))
 900     data = input_data.filter_data(table, continue_on_error=True)
 901
 902     # Prepare the header of the table
 903     try:
 904         header = [
 905             "Test case",
 906             "{0} Thput [Mpps]".format(table["reference"]["title"]),
 907             "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 908             "{0} Thput [Mpps]".format(table["compare"]["title"]),
 909             "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 910             "Delta [%]", "Stdev of delta [%]"]
 911         header_str = ",".join(header) + "\n"
 912     except (AttributeError, KeyError) as err:
 913         logging.error("The model is invalid, missing parameter: {0}".
 914                       format(err))
 915         return
 916
 917     # Create a list of available SOAK test results:
 918     tbl_dict = dict()
 919     for job, builds in table["compare"]["data"].items():
 920         for build in builds:
 921             for tst_name, tst_data in data[job][str(build)].iteritems():
 922                 if tst_data["type"] == "SOAK":
 923                     tst_name_mod = tst_name.replace("-soak", "")
 924                     if tbl_dict.get(tst_name_mod, None) is None:
 925                         groups = re.search(REGEX_NIC, tst_data["parent"])
 926                         nic = groups.group(0) if groups else ""
 927                         name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
 928                                                               split("-")[:-1]))
 929                         tbl_dict[tst_name_mod] = {
 930                             "name": name,
 931                             "ref-data": list(),
 932                             "cmp-data": list()
 933                         }
 934                     try:
 935                         tbl_dict[tst_name_mod]["cmp-data"].append(
 936                             tst_data["throughput"]["LOWER"])
 937                     except (KeyError, TypeError):
 938                         pass
 939     tests_lst = tbl_dict.keys()
 940
 941     # Add corresponding NDR test results:
 942     for job, builds in table["reference"]["data"].items():
 943         for build in builds:
 944             for tst_name, tst_data in data[job][str(build)].iteritems():
 945                 tst_name_mod = tst_name.replace("-ndrpdr", "").\
 946                     replace("-mrr", "")
 947                 if tst_name_mod in tests_lst:
 948                     try:
 949                         if tst_data["type"] in ("NDRPDR", "MRR", "BMRR"):
 950                             if table["include-tests"] == "MRR":
 951                                 result = tst_data["result"]["receive-rate"].avg
 952                             elif table["include-tests"] == "PDR":
 953                                 result = tst_data["throughput"]["PDR"]["LOWER"]
 954                             elif table["include-tests"] == "NDR":
 955                                 result = tst_data["throughput"]["NDR"]["LOWER"]
 956                             else:
 957                                 result = None
 958                             if result is not None:
 959                                 tbl_dict[tst_name_mod]["ref-data"].append(
 960                                     result)
 961                     except (KeyError, TypeError):
 962                         continue
 963
 964     tbl_lst = list()
 965     for tst_name in tbl_dict.keys():
 966         item = [tbl_dict[tst_name]["name"], ]
 967         data_r = tbl_dict[tst_name]["ref-data"]
 968         if data_r:
 969             data_r_mean = mean(data_r)
 970             item.append(round(data_r_mean / 1000000, 2))
 971             data_r_stdev = stdev(data_r)
 972             item.append(round(data_r_stdev / 1000000, 2))
 973         else:
 974             data_r_mean = None
 975             data_r_stdev = None
 976             item.extend([None, None])
 977         data_c = tbl_dict[tst_name]["cmp-data"]
 978         if data_c:
 979             data_c_mean = mean(data_c)
 980             item.append(round(data_c_mean / 1000000, 2))
 981             data_c_stdev = stdev(data_c)
 982             item.append(round(data_c_stdev / 1000000, 2))
 983         else:
 984             data_c_mean = None
 985             data_c_stdev = None
 986             item.extend([None, None])
 987         if data_r_mean and data_c_mean:
 988             delta, d_stdev = relative_change_stdev(
 989                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
 990             item.append(round(delta, 2))
 991             item.append(round(d_stdev, 2))
 992             tbl_lst.append(item)
 993
 994     # Sort the table according to the relative change
 995     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 996
 997     # Generate csv tables:
 998     csv_file = "{0}.csv".format(table["output-file"])
 999     with open(csv_file, "w") as file_handler:
1000         file_handler.write(header_str)
1001         for test in tbl_lst:
1002             file_handler.write(",".join([str(item) for item in test]) + "\n")
1003
1004     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
1005
1006
1007 def table_performance_trending_dashboard(table, input_data):
1008     """Generate the table(s) with algorithm:
1009     table_performance_trending_dashboard
1010     specified in the specification file.
1011
1012     :param table: Table to generate.
1013     :param input_data: Data to process.
1014     :type table: pandas.Series
1015     :type input_data: InputData
1016     """
1017
1018     logging.info("  Generating the table {0} ...".
1019                  format(table.get("title", "")))
1020
1021     # Transform the data
1022     logging.info("    Creating the data set for the {0} '{1}'.".
1023                  format(table.get("type", ""), table.get("title", "")))
1024     data = input_data.filter_data(table, continue_on_error=True)
1025
1026     # Prepare the header of the tables
1027     header = ["Test Case",
1028               "Trend [Mpps]",
1029               "Short-Term Change [%]",
1030               "Long-Term Change [%]",
1031               "Regressions [#]",
1032               "Progressions [#]"
1033               ]
1034     header_str = ",".join(header) + "\n"
1035
1036     # Prepare data to the table:
1037     tbl_dict = dict()
1038     for job, builds in table["data"].items():
1039         for build in builds:
1040             for tst_name, tst_data in data[job][str(build)].iteritems():
1041                 if tst_name.lower() in table.get("ignore-list", list()):
1042                     continue
1043                 if tbl_dict.get(tst_name, None) is None:
1044                     groups = re.search(REGEX_NIC, tst_data["parent"])
1045                     if not groups:
1046                         continue
1047                     nic = groups.group(0)
1048                     tbl_dict[tst_name] = {
1049                         "name": "{0}-{1}".format(nic, tst_data["name"]),
1050                         "data": OrderedDict()}
1051                 try:
1052                     tbl_dict[tst_name]["data"][str(build)] = \
1053                         tst_data["result"]["receive-rate"]
1054                 except (TypeError, KeyError):
1055                     pass  # No data in output.xml for this test
1056
1057     tbl_lst = list()
1058     for tst_name in tbl_dict.keys():
1059         data_t = tbl_dict[tst_name]["data"]
1060         if len(data_t) < 2:
1061             continue
1062
1063         classification_lst, avgs = classify_anomalies(data_t)
1064
1065         win_size = min(len(data_t), table["window"])
1066         long_win_size = min(len(data_t), table["long-trend-window"])
1067
1068         try:
1069             max_long_avg = max(
1070                 [x for x in avgs[-long_win_size:-win_size]
1071                  if not isnan(x)])
1072         except ValueError:
1073             max_long_avg = nan
1074         last_avg = avgs[-1]
1075         avg_week_ago = avgs[max(-win_size, -len(avgs))]
1076
1077         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
1078             rel_change_last = nan
1079         else:
1080             rel_change_last = round(
1081                 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
1082
1083         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
1084             rel_change_long = nan
1085         else:
1086             rel_change_long = round(
1087                 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
1088
1089         if classification_lst:
1090             if isnan(rel_change_last) and isnan(rel_change_long):
1091                 continue
1092             if (isnan(last_avg) or
1093                 isnan(rel_change_last) or
1094                 isnan(rel_change_long)):
1095                 continue
1096             tbl_lst.append(
1097                 [tbl_dict[tst_name]["name"],
1098                  round(last_avg / 1000000, 2),
1099                  rel_change_last,
1100                  rel_change_long,
1101                  classification_lst[-win_size:].count("regression"),
1102                  classification_lst[-win_size:].count("progression")])
1103
1104     tbl_lst.sort(key=lambda rel: rel[0])
1105
1106     tbl_sorted = list()
1107     for nrr in range(table["window"], -1, -1):
1108         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
1109         for nrp in range(table["window"], -1, -1):
1110             tbl_out = [item for item in tbl_reg if item[5] == nrp]
1111             tbl_out.sort(key=lambda rel: rel[2])
1112             tbl_sorted.extend(tbl_out)
1113
1114     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1115
1116     logging.info("    Writing file: '{0}'".format(file_name))
1117     with open(file_name, "w") as file_handler:
1118         file_handler.write(header_str)
1119         for test in tbl_sorted:
1120             file_handler.write(",".join([str(item) for item in test]) + '\n')
1121
1122     txt_file_name = "{0}.txt".format(table["output-file"])
1123     logging.info("    Writing file: '{0}'".format(txt_file_name))
1124     convert_csv_to_pretty_txt(file_name, txt_file_name)
1125
1126
1127 def _generate_url(base, testbed, test_name):
1128     """Generate URL to a trending plot from the name of the test case.
1129
1130     :param base: The base part of URL common to all test cases.
1131     :param testbed: The testbed used for testing.
1132     :param test_name: The name of the test case.
1133     :type base: str
1134     :type testbed: str
1135     :type test_name: str
1136     :returns: The URL to the plot with the trending data for the given test
1137         case.
1138     :rtype str
1139     """
1140
1141     url = base
1142     file_name = ""
1143     anchor = ".html#"
1144     feature = ""
1145
1146     if "lbdpdk" in test_name or "lbvpp" in test_name:
1147         file_name = "link_bonding"
1148
1149     elif "114b" in test_name and "vhost" in test_name:
1150         file_name = "vts"
1151
1152     elif "testpmd" in test_name or "l3fwd" in test_name:
1153         file_name = "dpdk"
1154
1155     elif "memif" in test_name:
1156         file_name = "container_memif"
1157         feature = "-base"
1158
1159     elif "srv6" in test_name:
1160         file_name = "srv6"
1161
1162     elif "vhost" in test_name:
1163         if "l2xcbase" in test_name or "l2bdbasemaclrn" in test_name:
1164             file_name = "vm_vhost_l2"
1165             if "114b" in test_name:
1166                 feature = ""
1167             elif "l2xcbase" in test_name and "x520" in test_name:
1168                 feature = "-base-l2xc"
1169             elif "l2bdbasemaclrn" in test_name and "x520" in test_name:
1170                 feature = "-base-l2bd"
1171             else:
1172                 feature = "-base"
1173         elif "ip4base" in test_name:
1174             file_name = "vm_vhost_ip4"
1175             feature = "-base"
1176
1177     elif "ipsecbasetnlsw" in test_name:
1178         file_name = "ipsecsw"
1179         feature = "-base-scale"
1180
1181     elif "ipsec" in test_name:
1182         file_name = "ipsec"
1183         feature = "-base-scale"
1184         if "hw-" in test_name:
1185             file_name = "ipsechw"
1186         elif "sw-" in test_name:
1187             file_name = "ipsecsw"
1188
1189     elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name:
1190         file_name = "ip4_tunnels"
1191         feature = "-base"
1192
1193     elif "ip4base" in test_name or "ip4scale" in test_name:
1194         file_name = "ip4"
1195         if "xl710" in test_name:
1196             feature = "-base-scale-features"
1197         elif "iacl" in test_name:
1198             feature = "-features-iacl"
1199         elif "oacl" in test_name:
1200             feature = "-features-oacl"
1201         elif "snat" in test_name or "cop" in test_name:
1202             feature = "-features"
1203         else:
1204             feature = "-base-scale"
1205
1206     elif "ip6base" in test_name or "ip6scale" in test_name:
1207         file_name = "ip6"
1208         feature = "-base-scale"
1209
1210     elif "l2xcbase" in test_name or "l2xcscale" in test_name \
1211             or "l2bdbasemaclrn" in test_name or "l2bdscale" in test_name \
1212             or "l2dbbasemaclrn" in test_name or "l2dbscale" in test_name:
1213         file_name = "l2"
1214         if "macip" in test_name:
1215             feature = "-features-macip"
1216         elif "iacl" in test_name:
1217             feature = "-features-iacl"
1218         elif "oacl" in test_name:
1219             feature = "-features-oacl"
1220         else:
1221             feature = "-base-scale"
1222
1223     if "x520" in test_name:
1224         nic = "x520-"
1225     elif "x710" in test_name:
1226         nic = "x710-"
1227     elif "xl710" in test_name:
1228         nic = "xl710-"
1229     elif "xxv710" in test_name:
1230         nic = "xxv710-"
1231     elif "vic1227" in test_name:
1232         nic = "vic1227-"
1233     elif "vic1385" in test_name:
1234         nic = "vic1385-"
1235     else:
1236         nic = ""
1237     anchor += nic
1238
1239     if "64b" in test_name:
1240         framesize = "64b"
1241     elif "78b" in test_name:
1242         framesize = "78b"
1243     elif "imix" in test_name:
1244         framesize = "imix"
1245     elif "9000b" in test_name:
1246         framesize = "9000b"
1247     elif "1518b" in test_name:
1248         framesize = "1518b"
1249     elif "114b" in test_name:
1250         framesize = "114b"
1251     else:
1252         framesize = ""
1253     anchor += framesize + '-'
1254
1255     if "1t1c" in test_name:
1256         anchor += "1t1c"
1257     elif "2t2c" in test_name:
1258         anchor += "2t2c"
1259     elif "4t4c" in test_name:
1260         anchor += "4t4c"
1261     elif "2t1c" in test_name:
1262         anchor += "2t1c"
1263     elif "4t2c" in test_name:
1264         anchor += "4t2c"
1265     elif "8t4c" in test_name:
1266         anchor += "8t4c"
1267
1268     return url + file_name + '-' + testbed + '-' + nic + framesize + \
1269         feature.replace("-int", "").replace("-tnl", "") + anchor + feature
1270
1271
1272 def table_performance_trending_dashboard_html(table, input_data):
1273     """Generate the table(s) with algorithm:
1274     table_performance_trending_dashboard_html specified in the specification
1275     file.
1276
1277     :param table: Table to generate.
1278     :param input_data: Data to process.
1279     :type table: dict
1280     :type input_data: InputData
1281     """
1282
1283     testbed = table.get("testbed", None)
1284     if testbed is None:
1285         logging.error("The testbed is not defined for the table '{0}'.".
1286                       format(table.get("title", "")))
1287         return
1288
1289     logging.info("  Generating the table {0} ...".
1290                  format(table.get("title", "")))
1291
1292     try:
1293         with open(table["input-file"], 'rb') as csv_file:
1294             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
1295             csv_lst = [item for item in csv_content]
1296     except KeyError:
1297         logging.warning("The input file is not defined.")
1298         return
1299     except csv.Error as err:
1300         logging.warning("Not possible to process the file '{0}'.\n{1}".
1301                         format(table["input-file"], err))
1302         return
1303
1304     # Table:
1305     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
1306
1307     # Table header:
1308     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
1309     for idx, item in enumerate(csv_lst[0]):
1310         alignment = "left" if idx == 0 else "center"
1311         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
1312         th.text = item
1313
1314     # Rows:
1315     colors = {"regression": ("#ffcccc", "#ff9999"),
1316               "progression": ("#c6ecc6", "#9fdf9f"),
1317               "normal": ("#e9f1fb", "#d4e4f7")}
1318     for r_idx, row in enumerate(csv_lst[1:]):
1319         if int(row[4]):
1320             color = "regression"
1321         elif int(row[5]):
1322             color = "progression"
1323         else:
1324             color = "normal"
1325         background = colors[color][r_idx % 2]
1326         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
1327
1328         # Columns:
1329         for c_idx, item in enumerate(row):
1330             alignment = "left" if c_idx == 0 else "center"
1331             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1332             # Name:
1333             if c_idx == 0:
1334                 url = _generate_url("../trending/", testbed, item)
1335                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1336                 ref.text = item
1337             else:
1338                 td.text = item
1339     try:
1340         with open(table["output-file"], 'w') as html_file:
1341             logging.info("    Writing file: '{0}'".format(table["output-file"]))
1342             html_file.write(".. raw:: html\n\n\t")
1343             html_file.write(ET.tostring(dashboard))
1344             html_file.write("\n\t<p><br><br></p>\n")
1345     except KeyError:
1346         logging.warning("The output file is not defined.")
1347         return
1348
1349
1350 def table_last_failed_tests(table, input_data):
1351     """Generate the table(s) with algorithm: table_last_failed_tests
1352     specified in the specification file.
1353
1354     :param table: Table to generate.
1355     :param input_data: Data to process.
1356     :type table: pandas.Series
1357     :type input_data: InputData
1358     """
1359
1360     logging.info("  Generating the table {0} ...".
1361                  format(table.get("title", "")))
1362
1363     # Transform the data
1364     logging.info("    Creating the data set for the {0} '{1}'.".
1365                  format(table.get("type", ""), table.get("title", "")))
1366     data = input_data.filter_data(table, continue_on_error=True)
1367
1368     if data is None or data.empty:
1369         logging.warn("    No data for the {0} '{1}'.".
1370                      format(table.get("type", ""), table.get("title", "")))
1371         return
1372
1373     tbl_list = list()
1374     for job, builds in table["data"].items():
1375         for build in builds:
1376             build = str(build)
1377             try:
1378                 version = input_data.metadata(job, build).get("version", "")
1379             except KeyError:
1380                 logging.error("Data for {job}: {build} is not present.".
1381                               format(job=job, build=build))
1382                 return
1383             tbl_list.append(build)
1384             tbl_list.append(version)
1385             for tst_name, tst_data in data[job][build].iteritems():
1386                 if tst_data["status"] != "FAIL":
1387                     continue
1388                 groups = re.search(REGEX_NIC, tst_data["parent"])
1389                 if not groups:
1390                     continue
1391                 nic = groups.group(0)
1392                 tbl_list.append("{0}-{1}".format(nic, tst_data["name"]))
1393
1394     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1395     logging.info("    Writing file: '{0}'".format(file_name))
1396     with open(file_name, "w") as file_handler:
1397         for test in tbl_list:
1398             file_handler.write(test + '\n')
1399
1400
1401 def table_failed_tests(table, input_data):
1402     """Generate the table(s) with algorithm: table_failed_tests
1403     specified in the specification file.
1404
1405     :param table: Table to generate.
1406     :param input_data: Data to process.
1407     :type table: pandas.Series
1408     :type input_data: InputData
1409     """
1410
1411     logging.info("  Generating the table {0} ...".
1412                  format(table.get("title", "")))
1413
1414     # Transform the data
1415     logging.info("    Creating the data set for the {0} '{1}'.".
1416                  format(table.get("type", ""), table.get("title", "")))
1417     data = input_data.filter_data(table, continue_on_error=True)
1418
1419     # Prepare the header of the tables
1420     header = ["Test Case",
1421               "Failures [#]",
1422               "Last Failure [Time]",
1423               "Last Failure [VPP-Build-Id]",
1424               "Last Failure [CSIT-Job-Build-Id]"]
1425
1426     # Generate the data for the table according to the model in the table
1427     # specification
1428
1429     now = dt.utcnow()
1430     timeperiod = timedelta(int(table.get("window", 7)))
1431
1432     tbl_dict = dict()
1433     for job, builds in table["data"].items():
1434         for build in builds:
1435             build = str(build)
1436             for tst_name, tst_data in data[job][build].iteritems():
1437                 if tst_name.lower() in table.get("ignore-list", list()):
1438                     continue
1439                 if tbl_dict.get(tst_name, None) is None:
1440                     groups = re.search(REGEX_NIC, tst_data["parent"])
1441                     if not groups:
1442                         continue
1443                     nic = groups.group(0)
1444                     tbl_dict[tst_name] = {
1445                         "name": "{0}-{1}".format(nic, tst_data["name"]),
1446                         "data": OrderedDict()}
1447                 try:
1448                     generated = input_data.metadata(job, build).\
1449                         get("generated", "")
1450                     if not generated:
1451                         continue
1452                     then = dt.strptime(generated, "%Y%m%d %H:%M")
1453                     if (now - then) <= timeperiod:
1454                         tbl_dict[tst_name]["data"][build] = (
1455                             tst_data["status"],
1456                             generated,
1457                             input_data.metadata(job, build).get("version", ""),
1458                             build)
1459                 except (TypeError, KeyError) as err:
1460                     logging.warning("tst_name: {} - err: {}".
1461                                     format(tst_name, repr(err)))
1462
1463     max_fails = 0
1464     tbl_lst = list()
1465     for tst_data in tbl_dict.values():
1466         fails_nr = 0
1467         for val in tst_data["data"].values():
1468             if val[0] == "FAIL":
1469                 fails_nr += 1
1470                 fails_last_date = val[1]
1471                 fails_last_vpp = val[2]
1472                 fails_last_csit = val[3]
1473         if fails_nr:
1474             max_fails = fails_nr if fails_nr > max_fails else max_fails
1475             tbl_lst.append([tst_data["name"],
1476                             fails_nr,
1477                             fails_last_date,
1478                             fails_last_vpp,
1479                             "mrr-daily-build-{0}".format(fails_last_csit)])
1480
1481     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1482     tbl_sorted = list()
1483     for nrf in range(max_fails, -1, -1):
1484         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1485         tbl_sorted.extend(tbl_fails)
1486     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1487
1488     logging.info("    Writing file: '{0}'".format(file_name))
1489     with open(file_name, "w") as file_handler:
1490         file_handler.write(",".join(header) + "\n")
1491         for test in tbl_sorted:
1492             file_handler.write(",".join([str(item) for item in test]) + '\n')
1493
1494     txt_file_name = "{0}.txt".format(table["output-file"])
1495     logging.info("    Writing file: '{0}'".format(txt_file_name))
1496     convert_csv_to_pretty_txt(file_name, txt_file_name)
1497
1498
1499 def table_failed_tests_html(table, input_data):
1500     """Generate the table(s) with algorithm: table_failed_tests_html
1501     specified in the specification file.
1502
1503     :param table: Table to generate.
1504     :param input_data: Data to process.
1505     :type table: pandas.Series
1506     :type input_data: InputData
1507     """
1508
1509     testbed = table.get("testbed", None)
1510     if testbed is None:
1511         logging.error("The testbed is not defined for the table '{0}'.".
1512                       format(table.get("title", "")))
1513         return
1514
1515     logging.info("  Generating the table {0} ...".
1516                  format(table.get("title", "")))
1517
1518     try:
1519         with open(table["input-file"], 'rb') as csv_file:
1520             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
1521             csv_lst = [item for item in csv_content]
1522     except KeyError:
1523         logging.warning("The input file is not defined.")
1524         return
1525     except csv.Error as err:
1526         logging.warning("Not possible to process the file '{0}'.\n{1}".
1527                         format(table["input-file"], err))
1528         return
1529
1530     # Table:
1531     failed_tests = ET.Element("table", attrib=dict(width="100%", border='0'))
1532
1533     # Table header:
1534     tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor="#7eade7"))
1535     for idx, item in enumerate(csv_lst[0]):
1536         alignment = "left" if idx == 0 else "center"
1537         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
1538         th.text = item
1539
1540     # Rows:
1541     colors = ("#e9f1fb", "#d4e4f7")
1542     for r_idx, row in enumerate(csv_lst[1:]):
1543         background = colors[r_idx % 2]
1544         tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor=background))
1545
1546         # Columns:
1547         for c_idx, item in enumerate(row):
1548             alignment = "left" if c_idx == 0 else "center"
1549             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1550             # Name:
1551             if c_idx == 0:
1552                 url = _generate_url("../trending/", testbed, item)
1553                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1554                 ref.text = item
1555             else:
1556                 td.text = item
1557     try:
1558         with open(table["output-file"], 'w') as html_file:
1559             logging.info("    Writing file: '{0}'".format(table["output-file"]))
1560             html_file.write(".. raw:: html\n\n\t")
1561             html_file.write(ET.tostring(failed_tests))
1562             html_file.write("\n\t<p><br><br></p>\n")
1563     except KeyError:
1564         logging.warning("The output file is not defined.")
1565         return