resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from string import replace
  23 from collections import OrderedDict
  24 from numpy import nan, isnan
  25 from xml.etree import ElementTree as ET
  26 from datetime import datetime as dt
  27 from datetime import timedelta
  28
  29 from utils import mean, stdev, relative_change, classify_anomalies, \
  30     convert_csv_to_pretty_txt, relative_change_stdev
  31
  32
  33 REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*')
  34
  35
  36 def generate_tables(spec, data):
  37     """Generate all tables specified in the specification file.
  38
  39     :param spec: Specification read from the specification file.
  40     :param data: Data to process.
  41     :type spec: Specification
  42     :type data: InputData
  43     """
  44
  45     logging.info("Generating the tables ...")
  46     for table in spec.tables:
  47         try:
  48             eval(table["algorithm"])(table, data)
  49         except NameError as err:
  50             logging.error("Probably algorithm '{alg}' is not defined: {err}".
  51                           format(alg=table["algorithm"], err=repr(err)))
  52     logging.info("Done.")
  53
  54
  55 def table_details(table, input_data):
  56     """Generate the table(s) with algorithm: table_detailed_test_results
  57     specified in the specification file.
  58
  59     :param table: Table to generate.
  60     :param input_data: Data to process.
  61     :type table: pandas.Series
  62     :type input_data: InputData
  63     """
  64
  65     logging.info("  Generating the table {0} ...".
  66                  format(table.get("title", "")))
  67
  68     # Transform the data
  69     logging.info("    Creating the data set for the {0} '{1}'.".
  70                  format(table.get("type", ""), table.get("title", "")))
  71     data = input_data.filter_data(table)
  72
  73     # Prepare the header of the tables
  74     header = list()
  75     for column in table["columns"]:
  76         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  77
  78     # Generate the data for the table according to the model in the table
  79     # specification
  80     job = table["data"].keys()[0]
  81     build = str(table["data"][job][0])
  82     try:
  83         suites = input_data.suites(job, build)
  84     except KeyError:
  85         logging.error("    No data available. The table will not be generated.")
  86         return
  87
  88     for suite_longname, suite in suites.iteritems():
  89         # Generate data
  90         suite_name = suite["name"]
  91         table_lst = list()
  92         for test in data[job][build].keys():
  93             if data[job][build][test]["parent"] in suite_name:
  94                 row_lst = list()
  95                 for column in table["columns"]:
  96                     try:
  97                         col_data = str(data[job][build][test][column["data"].
  98                                        split(" ")[1]]).replace('"', '""')
  99                         if column["data"].split(" ")[1] in ("conf-history",
 100                                                             "show-run"):
 101                             col_data = replace(col_data, " |br| ", "",
 102                                                maxreplace=1)
 103                             col_data = " |prein| {0} |preout| ".\
 104                                 format(col_data[:-5])
 105                         row_lst.append('"{0}"'.format(col_data))
 106                     except KeyError:
 107                         row_lst.append("No data")
 108                 table_lst.append(row_lst)
 109
 110         # Write the data to file
 111         if table_lst:
 112             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 113                                             table["output-file-ext"])
 114             logging.info("      Writing file: '{}'".format(file_name))
 115             with open(file_name, "w") as file_handler:
 116                 file_handler.write(",".join(header) + "\n")
 117                 for item in table_lst:
 118                     file_handler.write(",".join(item) + "\n")
 119
 120     logging.info("  Done.")
 121
 122
 123 def table_merged_details(table, input_data):
 124     """Generate the table(s) with algorithm: table_merged_details
 125     specified in the specification file.
 126
 127     :param table: Table to generate.
 128     :param input_data: Data to process.
 129     :type table: pandas.Series
 130     :type input_data: InputData
 131     """
 132
 133     logging.info("  Generating the table {0} ...".
 134                  format(table.get("title", "")))
 135
 136     # Transform the data
 137     logging.info("    Creating the data set for the {0} '{1}'.".
 138                  format(table.get("type", ""), table.get("title", "")))
 139     data = input_data.filter_data(table)
 140     data = input_data.merge_data(data)
 141     data.sort_index(inplace=True)
 142
 143     logging.info("    Creating the data set for the {0} '{1}'.".
 144                  format(table.get("type", ""), table.get("title", "")))
 145     suites = input_data.filter_data(table, data_set="suites")
 146     suites = input_data.merge_data(suites)
 147
 148     # Prepare the header of the tables
 149     header = list()
 150     for column in table["columns"]:
 151         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 152
 153     for _, suite in suites.iteritems():
 154         # Generate data
 155         suite_name = suite["name"]
 156         table_lst = list()
 157         for test in data.keys():
 158             if data[test]["parent"] in suite_name:
 159                 row_lst = list()
 160                 for column in table["columns"]:
 161                     try:
 162                         col_data = str(data[test][column["data"].
 163                                        split(" ")[1]]).replace('"', '""')
 164                         col_data = replace(col_data, "No Data",
 165                                            "Not Captured     ")
 166                         if column["data"].split(" ")[1] in ("conf-history",
 167                                                             "show-run"):
 168                             col_data = replace(col_data, " |br| ", "",
 169                                                maxreplace=1)
 170                             col_data = " |prein| {0} |preout| ".\
 171                                 format(col_data[:-5])
 172                         row_lst.append('"{0}"'.format(col_data))
 173                     except KeyError:
 174                         row_lst.append('"Not captured"')
 175                 table_lst.append(row_lst)
 176
 177         # Write the data to file
 178         if table_lst:
 179             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 180                                             table["output-file-ext"])
 181             logging.info("      Writing file: '{}'".format(file_name))
 182             with open(file_name, "w") as file_handler:
 183                 file_handler.write(",".join(header) + "\n")
 184                 for item in table_lst:
 185                     file_handler.write(",".join(item) + "\n")
 186
 187     logging.info("  Done.")
 188
 189
 190 def table_performance_comparison(table, input_data):
 191     """Generate the table(s) with algorithm: table_performance_comparison
 192     specified in the specification file.
 193
 194     :param table: Table to generate.
 195     :param input_data: Data to process.
 196     :type table: pandas.Series
 197     :type input_data: InputData
 198     """
 199
 200     logging.info("  Generating the table {0} ...".
 201                  format(table.get("title", "")))
 202
 203     # Transform the data
 204     logging.info("    Creating the data set for the {0} '{1}'.".
 205                  format(table.get("type", ""), table.get("title", "")))
 206     data = input_data.filter_data(table, continue_on_error=True)
 207
 208     # Prepare the header of the tables
 209     try:
 210         header = ["Test case", ]
 211
 212         if table["include-tests"] == "MRR":
 213             hdr_param = "Rec Rate"
 214         else:
 215             hdr_param = "Thput"
 216
 217         history = table.get("history", None)
 218         if history:
 219             for item in history:
 220                 header.extend(
 221                     ["{0} {1} [Mpps]".format(item["title"], hdr_param),
 222                      "{0} Stdev [Mpps]".format(item["title"])])
 223         header.extend(
 224             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
 225              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 226              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
 227              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 228              "Delta [%]"])
 229         header_str = ",".join(header) + "\n"
 230     except (AttributeError, KeyError) as err:
 231         logging.error("The model is invalid, missing parameter: {0}".
 232                       format(err))
 233         return
 234
 235     # Prepare data to the table:
 236     tbl_dict = dict()
 237     for job, builds in table["reference"]["data"].items():
 238         topo = "2n-skx" if "2n-skx" in job else ""
 239         for build in builds:
 240             for tst_name, tst_data in data[job][str(build)].iteritems():
 241                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
 242                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
 243                     replace("-ndrdisc", "").replace("-pdr", "").\
 244                     replace("-ndr", "").\
 245                     replace("1t1c", "1c").replace("2t1c", "1c").\
 246                     replace("2t2c", "2c").replace("4t2c", "2c").\
 247                     replace("4t4c", "4c").replace("8t4c", "4c")
 248                 if "across topologies" in table["title"].lower():
 249                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
 250                 if tbl_dict.get(tst_name_mod, None) is None:
 251                     groups = re.search(REGEX_NIC, tst_data["parent"])
 252                     nic = groups.group(0) if groups else ""
 253                     name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
 254                                                           split("-")[:-1]))
 255                     if "across testbeds" in table["title"].lower() or \
 256                             "across topologies" in table["title"].lower():
 257                         name = name.\
 258                             replace("1t1c", "1c").replace("2t1c", "1c").\
 259                             replace("2t2c", "2c").replace("4t2c", "2c").\
 260                             replace("4t4c", "4c").replace("8t4c", "4c")
 261                     tbl_dict[tst_name_mod] = {"name": name,
 262                                               "ref-data": list(),
 263                                               "cmp-data": list()}
 264                 try:
 265                     # TODO: Re-work when NDRPDRDISC tests are not used
 266                     if table["include-tests"] == "MRR":
 267                         tbl_dict[tst_name_mod]["ref-data"]. \
 268                             append(tst_data["result"]["receive-rate"].avg)
 269                     elif table["include-tests"] == "PDR":
 270                         if tst_data["type"] == "PDR":
 271                             tbl_dict[tst_name_mod]["ref-data"]. \
 272                                 append(tst_data["throughput"]["value"])
 273                         elif tst_data["type"] == "NDRPDR":
 274                             tbl_dict[tst_name_mod]["ref-data"].append(
 275                                 tst_data["throughput"]["PDR"]["LOWER"])
 276                     elif table["include-tests"] == "NDR":
 277                         if tst_data["type"] == "NDR":
 278                             tbl_dict[tst_name_mod]["ref-data"]. \
 279                                 append(tst_data["throughput"]["value"])
 280                         elif tst_data["type"] == "NDRPDR":
 281                             tbl_dict[tst_name_mod]["ref-data"].append(
 282                                 tst_data["throughput"]["NDR"]["LOWER"])
 283                     else:
 284                         continue
 285                 except TypeError:
 286                     pass  # No data in output.xml for this test
 287
 288     for job, builds in table["compare"]["data"].items():
 289         for build in builds:
 290             for tst_name, tst_data in data[job][str(build)].iteritems():
 291                 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
 292                     replace("-ndrpdr", "").replace("-pdrdisc", ""). \
 293                     replace("-ndrdisc", "").replace("-pdr", ""). \
 294                     replace("-ndr", "").\
 295                     replace("1t1c", "1c").replace("2t1c", "1c").\
 296                     replace("2t2c", "2c").replace("4t2c", "2c").\
 297                     replace("4t4c", "4c").replace("8t4c", "4c")
 298                 if "across topologies" in table["title"].lower():
 299                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
 300                 if tbl_dict.get(tst_name_mod, None) is None:
 301                     groups = re.search(REGEX_NIC, tst_data["parent"])
 302                     nic = groups.group(0) if groups else ""
 303                     name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
 304                                                           split("-")[:-1]))
 305                     if "across testbeds" in table["title"].lower() or \
 306                             "across topologies" in table["title"].lower():
 307                         name = name.\
 308                             replace("1t1c", "1c").replace("2t1c", "1c").\
 309                             replace("2t2c", "2c").replace("4t2c", "2c").\
 310                             replace("4t4c", "4c").replace("8t4c", "4c")
 311                     tbl_dict[tst_name_mod] = {"name": name,
 312                                               "ref-data": list(),
 313                                               "cmp-data": list()}
 314                 try:
 315                     # TODO: Re-work when NDRPDRDISC tests are not used
 316                     if table["include-tests"] == "MRR":
 317                         tbl_dict[tst_name_mod]["cmp-data"]. \
 318                             append(tst_data["result"]["receive-rate"].avg)
 319                     elif table["include-tests"] == "PDR":
 320                         if tst_data["type"] == "PDR":
 321                             tbl_dict[tst_name_mod]["cmp-data"]. \
 322                                 append(tst_data["throughput"]["value"])
 323                         elif tst_data["type"] == "NDRPDR":
 324                             tbl_dict[tst_name_mod]["cmp-data"].append(
 325                                 tst_data["throughput"]["PDR"]["LOWER"])
 326                     elif table["include-tests"] == "NDR":
 327                         if tst_data["type"] == "NDR":
 328                             tbl_dict[tst_name_mod]["cmp-data"]. \
 329                                 append(tst_data["throughput"]["value"])
 330                         elif tst_data["type"] == "NDRPDR":
 331                             tbl_dict[tst_name_mod]["cmp-data"].append(
 332                                 tst_data["throughput"]["NDR"]["LOWER"])
 333                     else:
 334                         continue
 335                 except (KeyError, TypeError):
 336                     pass
 337     if history:
 338         for item in history:
 339             for job, builds in item["data"].items():
 340                 for build in builds:
 341                     for tst_name, tst_data in data[job][str(build)].iteritems():
 342                         tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
 343                             replace("-ndrpdr", "").replace("-pdrdisc", ""). \
 344                             replace("-ndrdisc", "").replace("-pdr", ""). \
 345                             replace("-ndr", "").\
 346                             replace("1t1c", "1c").replace("2t1c", "1c").\
 347                             replace("2t2c", "2c").replace("4t2c", "2c").\
 348                             replace("4t4c", "4c").replace("8t4c", "4c")
 349                         if "across topologies" in table["title"].lower():
 350                             tst_name_mod = tst_name_mod.replace("2n1l-", "")
 351                         if tbl_dict.get(tst_name_mod, None) is None:
 352                             continue
 353                         if tbl_dict[tst_name_mod].get("history", None) is None:
 354                             tbl_dict[tst_name_mod]["history"] = OrderedDict()
 355                         if tbl_dict[tst_name_mod]["history"].get(item["title"],
 356                                                              None) is None:
 357                             tbl_dict[tst_name_mod]["history"][item["title"]] = \
 358                                 list()
 359                         try:
 360                             # TODO: Re-work when NDRPDRDISC tests are not used
 361                             if table["include-tests"] == "MRR":
 362                                 tbl_dict[tst_name_mod]["history"][item["title"
 363                                 ]].append(tst_data["result"]["receive-rate"].
 364                                           avg)
 365                             elif table["include-tests"] == "PDR":
 366                                 if tst_data["type"] == "PDR":
 367                                     tbl_dict[tst_name_mod]["history"][
 368                                         item["title"]].\
 369                                         append(tst_data["throughput"]["value"])
 370                                 elif tst_data["type"] == "NDRPDR":
 371                                     tbl_dict[tst_name_mod]["history"][item[
 372                                         "title"]].append(tst_data["throughput"][
 373                                         "PDR"]["LOWER"])
 374                             elif table["include-tests"] == "NDR":
 375                                 if tst_data["type"] == "NDR":
 376                                     tbl_dict[tst_name_mod]["history"][
 377                                         item["title"]].\
 378                                         append(tst_data["throughput"]["value"])
 379                                 elif tst_data["type"] == "NDRPDR":
 380                                     tbl_dict[tst_name_mod]["history"][item[
 381                                         "title"]].append(tst_data["throughput"][
 382                                         "NDR"]["LOWER"])
 383                             else:
 384                                 continue
 385                         except (TypeError, KeyError):
 386                             pass
 387
 388     tbl_lst = list()
 389     footnote = False
 390     for tst_name in tbl_dict.keys():
 391         item = [tbl_dict[tst_name]["name"], ]
 392         if history:
 393             if tbl_dict[tst_name].get("history", None) is not None:
 394                 for hist_data in tbl_dict[tst_name]["history"].values():
 395                     if hist_data:
 396                         item.append(round(mean(hist_data) / 1000000, 2))
 397                         item.append(round(stdev(hist_data) / 1000000, 2))
 398                     else:
 399                         item.extend(["Not tested", "Not tested"])
 400             else:
 401                 item.extend(["Not tested", "Not tested"])
 402         data_t = tbl_dict[tst_name]["ref-data"]
 403         if data_t:
 404             item.append(round(mean(data_t) / 1000000, 2))
 405             item.append(round(stdev(data_t) / 1000000, 2))
 406         else:
 407             item.extend(["Not tested", "Not tested"])
 408         data_t = tbl_dict[tst_name]["cmp-data"]
 409         if data_t:
 410             item.append(round(mean(data_t) / 1000000, 2))
 411             item.append(round(stdev(data_t) / 1000000, 2))
 412         else:
 413             item.extend(["Not tested", "Not tested"])
 414         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
 415             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 416         elif item[-4] is None or item[-2] is None or item[-4] == 0:
 417             item.append("New in CSIT-1908")
 418         elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]:
 419             item.append("See footnote [1]")
 420             footnote = True
 421         if (len(item) == len(header)) and (item[-3] is not None):
 422             tbl_lst.append(item)
 423
 424     # Sort the table according to the relative change
 425     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 426
 427     # Generate csv tables:
 428     csv_file = "{0}.csv".format(table["output-file"])
 429     with open(csv_file, "w") as file_handler:
 430         file_handler.write(header_str)
 431         for test in tbl_lst:
 432             file_handler.write(",".join([str(item) for item in test]) + "\n")
 433
 434     txt_file_name = "{0}.txt".format(table["output-file"])
 435     convert_csv_to_pretty_txt(csv_file, txt_file_name)
 436
 437     if footnote:
 438         with open(txt_file_name, 'a') as txt_file:
 439             txt_file.writelines([
 440                 "Footnotes:",
 441                 "[1] CSIT-1908 changed test methodology of dot1q tests in "
 442                 "2n-skx testbeds, dot1q encapsulation is now used on both "
 443                 "links of SUT.",
 444                 "    Previously dot1q was used only on a single link with the "
 445                 "other link carrying untagged Ethernet frames. This change "
 446                 "results",
 447                 "    in slightly lower throughput in CSIT-1908 for these "
 448                 "tests. See release notes."
 449             ])
 450
 451
 452 def table_performance_comparison_nic(table, input_data):
 453     """Generate the table(s) with algorithm: table_performance_comparison
 454     specified in the specification file.
 455
 456     :param table: Table to generate.
 457     :param input_data: Data to process.
 458     :type table: pandas.Series
 459     :type input_data: InputData
 460     """
 461
 462     logging.info("  Generating the table {0} ...".
 463                  format(table.get("title", "")))
 464
 465     # Transform the data
 466     logging.info("    Creating the data set for the {0} '{1}'.".
 467                  format(table.get("type", ""), table.get("title", "")))
 468     data = input_data.filter_data(table, continue_on_error=True)
 469
 470     # Prepare the header of the tables
 471     try:
 472         header = ["Test case", ]
 473
 474         if table["include-tests"] == "MRR":
 475             hdr_param = "Rec Rate"
 476         else:
 477             hdr_param = "Thput"
 478
 479         history = table.get("history", None)
 480         if history:
 481             for item in history:
 482                 header.extend(
 483                     ["{0} {1} [Mpps]".format(item["title"], hdr_param),
 484                      "{0} Stdev [Mpps]".format(item["title"])])
 485         header.extend(
 486             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
 487              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 488              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
 489              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 490              "Delta [%]"])
 491         header_str = ",".join(header) + "\n"
 492     except (AttributeError, KeyError) as err:
 493         logging.error("The model is invalid, missing parameter: {0}".
 494                       format(err))
 495         return
 496
 497     # Prepare data to the table:
 498     tbl_dict = dict()
 499     for job, builds in table["reference"]["data"].items():
 500         topo = "2n-skx" if "2n-skx" in job else ""
 501         for build in builds:
 502             for tst_name, tst_data in data[job][str(build)].iteritems():
 503                 if table["reference"]["nic"] not in tst_data["tags"]:
 504                     continue
 505                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
 506                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
 507                     replace("-ndrdisc", "").replace("-pdr", "").\
 508                     replace("-ndr", "").\
 509                     replace("1t1c", "1c").replace("2t1c", "1c").\
 510                     replace("2t2c", "2c").replace("4t2c", "2c").\
 511                     replace("4t4c", "4c").replace("8t4c", "4c")
 512                 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
 513                 if "across topologies" in table["title"].lower():
 514                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
 515                 if tbl_dict.get(tst_name_mod, None) is None:
 516                     name = "{0}".format("-".join(tst_data["name"].
 517                                                  split("-")[:-1]))
 518                     if "across testbeds" in table["title"].lower() or \
 519                             "across topologies" in table["title"].lower():
 520                         name = name.\
 521                             replace("1t1c", "1c").replace("2t1c", "1c").\
 522                             replace("2t2c", "2c").replace("4t2c", "2c").\
 523                             replace("4t4c", "4c").replace("8t4c", "4c")
 524                     tbl_dict[tst_name_mod] = {"name": name,
 525                                               "ref-data": list(),
 526                                               "cmp-data": list()}
 527                 try:
 528                     # TODO: Re-work when NDRPDRDISC tests are not used
 529                     if table["include-tests"] == "MRR":
 530                         tbl_dict[tst_name_mod]["ref-data"]. \
 531                             append(tst_data["result"]["receive-rate"].avg)
 532                     elif table["include-tests"] == "PDR":
 533                         if tst_data["type"] == "PDR":
 534                             tbl_dict[tst_name_mod]["ref-data"]. \
 535                                 append(tst_data["throughput"]["value"])
 536                         elif tst_data["type"] == "NDRPDR":
 537                             tbl_dict[tst_name_mod]["ref-data"].append(
 538                                 tst_data["throughput"]["PDR"]["LOWER"])
 539                     elif table["include-tests"] == "NDR":
 540                         if tst_data["type"] == "NDR":
 541                             tbl_dict[tst_name_mod]["ref-data"]. \
 542                                 append(tst_data["throughput"]["value"])
 543                         elif tst_data["type"] == "NDRPDR":
 544                             tbl_dict[tst_name_mod]["ref-data"].append(
 545                                 tst_data["throughput"]["NDR"]["LOWER"])
 546                     else:
 547                         continue
 548                 except TypeError:
 549                     pass  # No data in output.xml for this test
 550
 551     for job, builds in table["compare"]["data"].items():
 552         for build in builds:
 553             for tst_name, tst_data in data[job][str(build)].iteritems():
 554                 if table["compare"]["nic"] not in tst_data["tags"]:
 555                     continue
 556                 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
 557                     replace("-ndrpdr", "").replace("-pdrdisc", ""). \
 558                     replace("-ndrdisc", "").replace("-pdr", ""). \
 559                     replace("-ndr", "").\
 560                     replace("1t1c", "1c").replace("2t1c", "1c").\
 561                     replace("2t2c", "2c").replace("4t2c", "2c").\
 562                     replace("4t4c", "4c").replace("8t4c", "4c")
 563                 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
 564                 if "across topologies" in table["title"].lower():
 565                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
 566                 if tbl_dict.get(tst_name_mod, None) is None:
 567                     name = "{0}".format("-".join(tst_data["name"].
 568                                                  split("-")[:-1]))
 569                     if "across testbeds" in table["title"].lower() or \
 570                             "across topologies" in table["title"].lower():
 571                         name = name.\
 572                             replace("1t1c", "1c").replace("2t1c", "1c").\
 573                             replace("2t2c", "2c").replace("4t2c", "2c").\
 574                             replace("4t4c", "4c").replace("8t4c", "4c")
 575                     tbl_dict[tst_name_mod] = {"name": name,
 576                                               "ref-data": list(),
 577                                               "cmp-data": list()}
 578                 try:
 579                     # TODO: Re-work when NDRPDRDISC tests are not used
 580                     if table["include-tests"] == "MRR":
 581                         tbl_dict[tst_name_mod]["cmp-data"]. \
 582                             append(tst_data["result"]["receive-rate"].avg)
 583                     elif table["include-tests"] == "PDR":
 584                         if tst_data["type"] == "PDR":
 585                             tbl_dict[tst_name_mod]["cmp-data"]. \
 586                                 append(tst_data["throughput"]["value"])
 587                         elif tst_data["type"] == "NDRPDR":
 588                             tbl_dict[tst_name_mod]["cmp-data"].append(
 589                                 tst_data["throughput"]["PDR"]["LOWER"])
 590                     elif table["include-tests"] == "NDR":
 591                         if tst_data["type"] == "NDR":
 592                             tbl_dict[tst_name_mod]["cmp-data"]. \
 593                                 append(tst_data["throughput"]["value"])
 594                         elif tst_data["type"] == "NDRPDR":
 595                             tbl_dict[tst_name_mod]["cmp-data"].append(
 596                                 tst_data["throughput"]["NDR"]["LOWER"])
 597                     else:
 598                         continue
 599                 except (KeyError, TypeError):
 600                     pass
 601
 602     if history:
 603         for item in history:
 604             for job, builds in item["data"].items():
 605                 for build in builds:
 606                     for tst_name, tst_data in data[job][str(build)].iteritems():
 607                         if item["nic"] not in tst_data["tags"]:
 608                             continue
 609                         tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
 610                             replace("-ndrpdr", "").replace("-pdrdisc", ""). \
 611                             replace("-ndrdisc", "").replace("-pdr", ""). \
 612                             replace("-ndr", "").\
 613                             replace("1t1c", "1c").replace("2t1c", "1c").\
 614                             replace("2t2c", "2c").replace("4t2c", "2c").\
 615                             replace("4t4c", "4c").replace("8t4c", "4c")
 616                         tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
 617                         if "across topologies" in table["title"].lower():
 618                             tst_name_mod = tst_name_mod.replace("2n1l-", "")
 619                         if tbl_dict.get(tst_name_mod, None) is None:
 620                             continue
 621                         if tbl_dict[tst_name_mod].get("history", None) is None:
 622                             tbl_dict[tst_name_mod]["history"] = OrderedDict()
 623                         if tbl_dict[tst_name_mod]["history"].get(item["title"],
 624                                                              None) is None:
 625                             tbl_dict[tst_name_mod]["history"][item["title"]] = \
 626                                 list()
 627                         try:
 628                             # TODO: Re-work when NDRPDRDISC tests are not used
 629                             if table["include-tests"] == "MRR":
 630                                 tbl_dict[tst_name_mod]["history"][item["title"
 631                                 ]].append(tst_data["result"]["receive-rate"].
 632                                           avg)
 633                             elif table["include-tests"] == "PDR":
 634                                 if tst_data["type"] == "PDR":
 635                                     tbl_dict[tst_name_mod]["history"][
 636                                         item["title"]].\
 637                                         append(tst_data["throughput"]["value"])
 638                                 elif tst_data["type"] == "NDRPDR":
 639                                     tbl_dict[tst_name_mod]["history"][item[
 640                                         "title"]].append(tst_data["throughput"][
 641                                         "PDR"]["LOWER"])
 642                             elif table["include-tests"] == "NDR":
 643                                 if tst_data["type"] == "NDR":
 644                                     tbl_dict[tst_name_mod]["history"][
 645                                         item["title"]].\
 646                                         append(tst_data["throughput"]["value"])
 647                                 elif tst_data["type"] == "NDRPDR":
 648                                     tbl_dict[tst_name_mod]["history"][item[
 649                                         "title"]].append(tst_data["throughput"][
 650                                         "NDR"]["LOWER"])
 651                             else:
 652                                 continue
 653                         except (TypeError, KeyError):
 654                             pass
 655
 656     tbl_lst = list()
 657     footnote = False
 658     for tst_name in tbl_dict.keys():
 659         item = [tbl_dict[tst_name]["name"], ]
 660         if history:
 661             if tbl_dict[tst_name].get("history", None) is not None:
 662                 for hist_data in tbl_dict[tst_name]["history"].values():
 663                     if hist_data:
 664                         item.append(round(mean(hist_data) / 1000000, 2))
 665                         item.append(round(stdev(hist_data) / 1000000, 2))
 666                     else:
 667                         item.extend(["Not tested", "Not tested"])
 668             else:
 669                 item.extend(["Not tested", "Not tested"])
 670         data_t = tbl_dict[tst_name]["ref-data"]
 671         if data_t:
 672             item.append(round(mean(data_t) / 1000000, 2))
 673             item.append(round(stdev(data_t) / 1000000, 2))
 674         else:
 675             item.extend(["Not tested", "Not tested"])
 676         data_t = tbl_dict[tst_name]["cmp-data"]
 677         if data_t:
 678             item.append(round(mean(data_t) / 1000000, 2))
 679             item.append(round(stdev(data_t) / 1000000, 2))
 680         else:
 681             item.extend(["Not tested", "Not tested"])
 682         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
 683             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 684         elif item[-4] is None or item[-2] is None or item[-4] == 0:
 685             item.append("New in CSIT-1908")
 686         elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]:
 687             item.append("See footnote [1]")
 688             footnote = True
 689         if (len(item) == len(header)) and (item[-3] is not None):
 690             tbl_lst.append(item)
 691
 692     # Sort the table according to the relative change
 693     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 694
 695     # Generate csv tables:
 696     csv_file = "{0}.csv".format(table["output-file"])
 697     with open(csv_file, "w") as file_handler:
 698         file_handler.write(header_str)
 699         for test in tbl_lst:
 700             file_handler.write(",".join([str(item) for item in test]) + "\n")
 701
 702     txt_file_name = "{0}.txt".format(table["output-file"])
 703     convert_csv_to_pretty_txt(csv_file, txt_file_name)
 704
 705     if footnote:
 706         with open(txt_file_name, 'a') as txt_file:
 707             txt_file.writelines([
 708                 "Footnotes:",
 709                 "[1] CSIT-1908 changed test methodology of dot1q tests in "
 710                 "2n-skx testbeds, dot1q encapsulation is now used on both "
 711                 "links of SUT.",
 712                 "    Previously dot1q was used only on a single link with the "
 713                 "other link carrying untagged Ethernet frames. This change "
 714                 "results",
 715                 "    in slightly lower throughput in CSIT-1908 for these "
 716                 "tests. See release notes."
 717             ])
 718
 719
 720 def table_nics_comparison(table, input_data):
 721     """Generate the table(s) with algorithm: table_nics_comparison
 722     specified in the specification file.
 723
 724     :param table: Table to generate.
 725     :param input_data: Data to process.
 726     :type table: pandas.Series
 727     :type input_data: InputData
 728     """
 729
 730     logging.info("  Generating the table {0} ...".
 731                  format(table.get("title", "")))
 732
 733     # Transform the data
 734     logging.info("    Creating the data set for the {0} '{1}'.".
 735                  format(table.get("type", ""), table.get("title", "")))
 736     data = input_data.filter_data(table, continue_on_error=True)
 737
 738     # Prepare the header of the tables
 739     try:
 740         header = ["Test case", ]
 741
 742         if table["include-tests"] == "MRR":
 743             hdr_param = "Receive Rate"
 744         else:
 745             hdr_param = "Throughput"
 746
 747         header.extend(
 748             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
 749              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 750              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
 751              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 752              "Delta [%]"])
 753         header_str = ",".join(header) + "\n"
 754     except (AttributeError, KeyError) as err:
 755         logging.error("The model is invalid, missing parameter: {0}".
 756                       format(err))
 757         return
 758
 759     # Prepare data to the table:
 760     tbl_dict = dict()
 761     for job, builds in table["data"].items():
 762         for build in builds:
 763             for tst_name, tst_data in data[job][str(build)].iteritems():
 764                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
 765                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
 766                     replace("-ndrdisc", "").replace("-pdr", "").\
 767                     replace("-ndr", "").\
 768                     replace("1t1c", "1c").replace("2t1c", "1c").\
 769                     replace("2t2c", "2c").replace("4t2c", "2c").\
 770                     replace("4t4c", "4c").replace("8t4c", "4c")
 771                 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
 772                 if tbl_dict.get(tst_name_mod, None) is None:
 773                     name = "-".join(tst_data["name"].split("-")[:-1])
 774                     tbl_dict[tst_name_mod] = {"name": name,
 775                                               "ref-data": list(),
 776                                               "cmp-data": list()}
 777                 try:
 778                     if table["include-tests"] == "MRR":
 779                         result = tst_data["result"]["receive-rate"].avg
 780                     elif table["include-tests"] == "PDR":
 781                         result = tst_data["throughput"]["PDR"]["LOWER"]
 782                     elif table["include-tests"] == "NDR":
 783                         result = tst_data["throughput"]["NDR"]["LOWER"]
 784                     else:
 785                         result = None
 786
 787                     if result:
 788                         if table["reference"]["nic"] in tst_data["tags"]:
 789                             tbl_dict[tst_name_mod]["ref-data"].append(result)
 790                         elif table["compare"]["nic"] in tst_data["tags"]:
 791                             tbl_dict[tst_name_mod]["cmp-data"].append(result)
 792                 except (TypeError, KeyError) as err:
 793                     logging.debug("No data for {0}".format(tst_name))
 794                     logging.debug(repr(err))
 795                     # No data in output.xml for this test
 796
 797     tbl_lst = list()
 798     for tst_name in tbl_dict.keys():
 799         item = [tbl_dict[tst_name]["name"], ]
 800         data_t = tbl_dict[tst_name]["ref-data"]
 801         if data_t:
 802             item.append(round(mean(data_t) / 1000000, 2))
 803             item.append(round(stdev(data_t) / 1000000, 2))
 804         else:
 805             item.extend([None, None])
 806         data_t = tbl_dict[tst_name]["cmp-data"]
 807         if data_t:
 808             item.append(round(mean(data_t) / 1000000, 2))
 809             item.append(round(stdev(data_t) / 1000000, 2))
 810         else:
 811             item.extend([None, None])
 812         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
 813             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 814         if len(item) == len(header):
 815             tbl_lst.append(item)
 816
 817     # Sort the table according to the relative change
 818     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 819
 820     # Generate csv tables:
 821     csv_file = "{0}.csv".format(table["output-file"])
 822     with open(csv_file, "w") as file_handler:
 823         file_handler.write(header_str)
 824         for test in tbl_lst:
 825             file_handler.write(",".join([str(item) for item in test]) + "\n")
 826
 827     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
 828
 829
 830 def table_soak_vs_ndr(table, input_data):
 831     """Generate the table(s) with algorithm: table_soak_vs_ndr
 832     specified in the specification file.
 833
 834     :param table: Table to generate.
 835     :param input_data: Data to process.
 836     :type table: pandas.Series
 837     :type input_data: InputData
 838     """
 839
 840     logging.info("  Generating the table {0} ...".
 841                  format(table.get("title", "")))
 842
 843     # Transform the data
 844     logging.info("    Creating the data set for the {0} '{1}'.".
 845                  format(table.get("type", ""), table.get("title", "")))
 846     data = input_data.filter_data(table, continue_on_error=True)
 847
 848     # Prepare the header of the table
 849     try:
 850         header = [
 851             "Test case",
 852             "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 853             "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 854             "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 855             "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 856             "Delta [%]", "Stdev of delta [%]"]
 857         header_str = ",".join(header) + "\n"
 858     except (AttributeError, KeyError) as err:
 859         logging.error("The model is invalid, missing parameter: {0}".
 860                       format(err))
 861         return
 862
 863     # Create a list of available SOAK test results:
 864     tbl_dict = dict()
 865     for job, builds in table["compare"]["data"].items():
 866         for build in builds:
 867             for tst_name, tst_data in data[job][str(build)].iteritems():
 868                 if tst_data["type"] == "SOAK":
 869                     tst_name_mod = tst_name.replace("-soak", "")
 870                     if tbl_dict.get(tst_name_mod, None) is None:
 871                         groups = re.search(REGEX_NIC, tst_data["parent"])
 872                         nic = groups.group(0) if groups else ""
 873                         name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
 874                                                               split("-")[:-1]))
 875                         tbl_dict[tst_name_mod] = {
 876                             "name": name,
 877                             "ref-data": list(),
 878                             "cmp-data": list()
 879                         }
 880                     try:
 881                         tbl_dict[tst_name_mod]["cmp-data"].append(
 882                             tst_data["throughput"]["LOWER"])
 883                     except (KeyError, TypeError):
 884                         pass
 885     tests_lst = tbl_dict.keys()
 886
 887     # Add corresponding NDR test results:
 888     for job, builds in table["reference"]["data"].items():
 889         for build in builds:
 890             for tst_name, tst_data in data[job][str(build)].iteritems():
 891                 tst_name_mod = tst_name.replace("-ndrpdr", "").\
 892                     replace("-mrr", "")
 893                 if tst_name_mod in tests_lst:
 894                     try:
 895                         if tst_data["type"] in ("NDRPDR", "MRR", "BMRR"):
 896                             if table["include-tests"] == "MRR":
 897                                 result = tst_data["result"]["receive-rate"].avg
 898                             elif table["include-tests"] == "PDR":
 899                                 result = tst_data["throughput"]["PDR"]["LOWER"]
 900                             elif table["include-tests"] == "NDR":
 901                                 result = tst_data["throughput"]["NDR"]["LOWER"]
 902                             else:
 903                                 result = None
 904                             if result is not None:
 905                                 tbl_dict[tst_name_mod]["ref-data"].append(
 906                                     result)
 907                     except (KeyError, TypeError):
 908                         continue
 909
 910     tbl_lst = list()
 911     for tst_name in tbl_dict.keys():
 912         item = [tbl_dict[tst_name]["name"], ]
 913         data_r = tbl_dict[tst_name]["ref-data"]
 914         if data_r:
 915             data_r_mean = mean(data_r)
 916             item.append(round(data_r_mean / 1000000, 2))
 917             data_r_stdev = stdev(data_r)
 918             item.append(round(data_r_stdev / 1000000, 2))
 919         else:
 920             data_r_mean = None
 921             data_r_stdev = None
 922             item.extend([None, None])
 923         data_c = tbl_dict[tst_name]["cmp-data"]
 924         if data_c:
 925             data_c_mean = mean(data_c)
 926             item.append(round(data_c_mean / 1000000, 2))
 927             data_c_stdev = stdev(data_c)
 928             item.append(round(data_c_stdev / 1000000, 2))
 929         else:
 930             data_c_mean = None
 931             data_c_stdev = None
 932             item.extend([None, None])
 933         if data_r_mean and data_c_mean:
 934             delta, d_stdev = relative_change_stdev(
 935                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
 936             item.append(round(delta, 2))
 937             item.append(round(d_stdev, 2))
 938             tbl_lst.append(item)
 939
 940     # Sort the table according to the relative change
 941     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 942
 943     # Generate csv tables:
 944     csv_file = "{0}.csv".format(table["output-file"])
 945     with open(csv_file, "w") as file_handler:
 946         file_handler.write(header_str)
 947         for test in tbl_lst:
 948             file_handler.write(",".join([str(item) for item in test]) + "\n")
 949
 950     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
 951
 952
 953 def table_performance_trending_dashboard(table, input_data):
 954     """Generate the table(s) with algorithm:
 955     table_performance_trending_dashboard
 956     specified in the specification file.
 957
 958     :param table: Table to generate.
 959     :param input_data: Data to process.
 960     :type table: pandas.Series
 961     :type input_data: InputData
 962     """
 963
 964     logging.info("  Generating the table {0} ...".
 965                  format(table.get("title", "")))
 966
 967     # Transform the data
 968     logging.info("    Creating the data set for the {0} '{1}'.".
 969                  format(table.get("type", ""), table.get("title", "")))
 970     data = input_data.filter_data(table, continue_on_error=True)
 971
 972     # Prepare the header of the tables
 973     header = ["Test Case",
 974               "Trend [Mpps]",
 975               "Short-Term Change [%]",
 976               "Long-Term Change [%]",
 977               "Regressions [#]",
 978               "Progressions [#]"
 979               ]
 980     header_str = ",".join(header) + "\n"
 981
 982     # Prepare data to the table:
 983     tbl_dict = dict()
 984     for job, builds in table["data"].items():
 985         for build in builds:
 986             for tst_name, tst_data in data[job][str(build)].iteritems():
 987                 if tst_name.lower() in table.get("ignore-list", list()):
 988                     continue
 989                 if tbl_dict.get(tst_name, None) is None:
 990                     groups = re.search(REGEX_NIC, tst_data["parent"])
 991                     if not groups:
 992                         continue
 993                     nic = groups.group(0)
 994                     tbl_dict[tst_name] = {
 995                         "name": "{0}-{1}".format(nic, tst_data["name"]),
 996                         "data": OrderedDict()}
 997                 try:
 998                     tbl_dict[tst_name]["data"][str(build)] = \
 999                         tst_data["result"]["receive-rate"]
1000                 except (TypeError, KeyError):
1001                     pass  # No data in output.xml for this test
1002
1003     tbl_lst = list()
1004     for tst_name in tbl_dict.keys():
1005         data_t = tbl_dict[tst_name]["data"]
1006         if len(data_t) < 2:
1007             continue
1008
1009         classification_lst, avgs = classify_anomalies(data_t)
1010
1011         win_size = min(len(data_t), table["window"])
1012         long_win_size = min(len(data_t), table["long-trend-window"])
1013
1014         try:
1015             max_long_avg = max(
1016                 [x for x in avgs[-long_win_size:-win_size]
1017                  if not isnan(x)])
1018         except ValueError:
1019             max_long_avg = nan
1020         last_avg = avgs[-1]
1021         avg_week_ago = avgs[max(-win_size, -len(avgs))]
1022
1023         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
1024             rel_change_last = nan
1025         else:
1026             rel_change_last = round(
1027                 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
1028
1029         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
1030             rel_change_long = nan
1031         else:
1032             rel_change_long = round(
1033                 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
1034
1035         if classification_lst:
1036             if isnan(rel_change_last) and isnan(rel_change_long):
1037                 continue
1038             if (isnan(last_avg) or
1039                 isnan(rel_change_last) or
1040                 isnan(rel_change_long)):
1041                 continue
1042             tbl_lst.append(
1043                 [tbl_dict[tst_name]["name"],
1044                  round(last_avg / 1000000, 2),
1045                  rel_change_last,
1046                  rel_change_long,
1047                  classification_lst[-win_size:].count("regression"),
1048                  classification_lst[-win_size:].count("progression")])
1049
1050     tbl_lst.sort(key=lambda rel: rel[0])
1051
1052     tbl_sorted = list()
1053     for nrr in range(table["window"], -1, -1):
1054         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
1055         for nrp in range(table["window"], -1, -1):
1056             tbl_out = [item for item in tbl_reg if item[5] == nrp]
1057             tbl_out.sort(key=lambda rel: rel[2])
1058             tbl_sorted.extend(tbl_out)
1059
1060     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1061
1062     logging.info("    Writing file: '{0}'".format(file_name))
1063     with open(file_name, "w") as file_handler:
1064         file_handler.write(header_str)
1065         for test in tbl_sorted:
1066             file_handler.write(",".join([str(item) for item in test]) + '\n')
1067
1068     txt_file_name = "{0}.txt".format(table["output-file"])
1069     logging.info("    Writing file: '{0}'".format(txt_file_name))
1070     convert_csv_to_pretty_txt(file_name, txt_file_name)
1071
1072
1073 def _generate_url(base, testbed, test_name):
1074     """Generate URL to a trending plot from the name of the test case.
1075
1076     :param base: The base part of URL common to all test cases.
1077     :param testbed: The testbed used for testing.
1078     :param test_name: The name of the test case.
1079     :type base: str
1080     :type testbed: str
1081     :type test_name: str
1082     :returns: The URL to the plot with the trending data for the given test
1083         case.
1084     :rtype str
1085     """
1086
1087     url = base
1088     file_name = ""
1089     anchor = ".html#"
1090     feature = ""
1091
1092     if "lbdpdk" in test_name or "lbvpp" in test_name:
1093         file_name = "link_bonding"
1094
1095     elif "114b" in test_name and "vhost" in test_name:
1096         file_name = "vts"
1097
1098     elif "testpmd" in test_name or "l3fwd" in test_name:
1099         file_name = "dpdk"
1100
1101     elif "memif" in test_name:
1102         file_name = "container_memif"
1103         feature = "-base"
1104
1105     elif "srv6" in test_name:
1106         file_name = "srv6"
1107
1108     elif "vhost" in test_name:
1109         if "l2xcbase" in test_name or "l2bdbasemaclrn" in test_name:
1110             file_name = "vm_vhost_l2"
1111             if "114b" in test_name:
1112                 feature = ""
1113             elif "l2xcbase" in test_name and "x520" in test_name:
1114                 feature = "-base-l2xc"
1115             elif "l2bdbasemaclrn" in test_name and "x520" in test_name:
1116                 feature = "-base-l2bd"
1117             else:
1118                 feature = "-base"
1119         elif "ip4base" in test_name:
1120             file_name = "vm_vhost_ip4"
1121             feature = "-base"
1122
1123     elif "ipsecbasetnlsw" in test_name:
1124         file_name = "ipsecsw"
1125         feature = "-base-scale"
1126
1127     elif "ipsec" in test_name:
1128         file_name = "ipsec"
1129         feature = "-base-scale"
1130         if "hw-" in test_name:
1131             file_name = "ipsechw"
1132         elif "sw-" in test_name:
1133             file_name = "ipsecsw"
1134
1135     elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name:
1136         file_name = "ip4_tunnels"
1137         feature = "-base"
1138
1139     elif "ip4base" in test_name or "ip4scale" in test_name:
1140         file_name = "ip4"
1141         if "xl710" in test_name:
1142             feature = "-base-scale-features"
1143         elif "iacl" in test_name:
1144             feature = "-features-iacl"
1145         elif "oacl" in test_name:
1146             feature = "-features-oacl"
1147         elif "snat" in test_name or "cop" in test_name:
1148             feature = "-features"
1149         else:
1150             feature = "-base-scale"
1151
1152     elif "ip6base" in test_name or "ip6scale" in test_name:
1153         file_name = "ip6"
1154         feature = "-base-scale"
1155
1156     elif "l2xcbase" in test_name or "l2xcscale" in test_name \
1157             or "l2bdbasemaclrn" in test_name or "l2bdscale" in test_name \
1158             or "l2dbbasemaclrn" in test_name or "l2dbscale" in test_name:
1159         file_name = "l2"
1160         if "macip" in test_name:
1161             feature = "-features-macip"
1162         elif "iacl" in test_name:
1163             feature = "-features-iacl"
1164         elif "oacl" in test_name:
1165             feature = "-features-oacl"
1166         else:
1167             feature = "-base-scale"
1168
1169     if "x520" in test_name:
1170         nic = "x520-"
1171     elif "x710" in test_name:
1172         nic = "x710-"
1173     elif "xl710" in test_name:
1174         nic = "xl710-"
1175     elif "xxv710" in test_name:
1176         nic = "xxv710-"
1177     elif "vic1227" in test_name:
1178         nic = "vic1227-"
1179     elif "vic1385" in test_name:
1180         nic = "vic1385-"
1181     else:
1182         nic = ""
1183     anchor += nic
1184
1185     if "64b" in test_name:
1186         framesize = "64b"
1187     elif "78b" in test_name:
1188         framesize = "78b"
1189     elif "imix" in test_name:
1190         framesize = "imix"
1191     elif "9000b" in test_name:
1192         framesize = "9000b"
1193     elif "1518b" in test_name:
1194         framesize = "1518b"
1195     elif "114b" in test_name:
1196         framesize = "114b"
1197     else:
1198         framesize = ""
1199     anchor += framesize + '-'
1200
1201     if "1t1c" in test_name:
1202         anchor += "1t1c"
1203     elif "2t2c" in test_name:
1204         anchor += "2t2c"
1205     elif "4t4c" in test_name:
1206         anchor += "4t4c"
1207     elif "2t1c" in test_name:
1208         anchor += "2t1c"
1209     elif "4t2c" in test_name:
1210         anchor += "4t2c"
1211     elif "8t4c" in test_name:
1212         anchor += "8t4c"
1213
1214     return url + file_name + '-' + testbed + '-' + nic + framesize + \
1215         feature.replace("-int", "").replace("-tnl", "") + anchor + feature
1216
1217
1218 def table_performance_trending_dashboard_html(table, input_data):
1219     """Generate the table(s) with algorithm:
1220     table_performance_trending_dashboard_html specified in the specification
1221     file.
1222
1223     :param table: Table to generate.
1224     :param input_data: Data to process.
1225     :type table: dict
1226     :type input_data: InputData
1227     """
1228
1229     testbed = table.get("testbed", None)
1230     if testbed is None:
1231         logging.error("The testbed is not defined for the table '{0}'.".
1232                       format(table.get("title", "")))
1233         return
1234
1235     logging.info("  Generating the table {0} ...".
1236                  format(table.get("title", "")))
1237
1238     try:
1239         with open(table["input-file"], 'rb') as csv_file:
1240             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
1241             csv_lst = [item for item in csv_content]
1242     except KeyError:
1243         logging.warning("The input file is not defined.")
1244         return
1245     except csv.Error as err:
1246         logging.warning("Not possible to process the file '{0}'.\n{1}".
1247                         format(table["input-file"], err))
1248         return
1249
1250     # Table:
1251     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
1252
1253     # Table header:
1254     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
1255     for idx, item in enumerate(csv_lst[0]):
1256         alignment = "left" if idx == 0 else "center"
1257         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
1258         th.text = item
1259
1260     # Rows:
1261     colors = {"regression": ("#ffcccc", "#ff9999"),
1262               "progression": ("#c6ecc6", "#9fdf9f"),
1263               "normal": ("#e9f1fb", "#d4e4f7")}
1264     for r_idx, row in enumerate(csv_lst[1:]):
1265         if int(row[4]):
1266             color = "regression"
1267         elif int(row[5]):
1268             color = "progression"
1269         else:
1270             color = "normal"
1271         background = colors[color][r_idx % 2]
1272         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
1273
1274         # Columns:
1275         for c_idx, item in enumerate(row):
1276             alignment = "left" if c_idx == 0 else "center"
1277             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1278             # Name:
1279             if c_idx == 0:
1280                 url = _generate_url("../trending/", testbed, item)
1281                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1282                 ref.text = item
1283             else:
1284                 td.text = item
1285     try:
1286         with open(table["output-file"], 'w') as html_file:
1287             logging.info("    Writing file: '{0}'".format(table["output-file"]))
1288             html_file.write(".. raw:: html\n\n\t")
1289             html_file.write(ET.tostring(dashboard))
1290             html_file.write("\n\t<p><br><br></p>\n")
1291     except KeyError:
1292         logging.warning("The output file is not defined.")
1293         return
1294
1295
1296 def table_last_failed_tests(table, input_data):
1297     """Generate the table(s) with algorithm: table_last_failed_tests
1298     specified in the specification file.
1299
1300     :param table: Table to generate.
1301     :param input_data: Data to process.
1302     :type table: pandas.Series
1303     :type input_data: InputData
1304     """
1305
1306     logging.info("  Generating the table {0} ...".
1307                  format(table.get("title", "")))
1308
1309     # Transform the data
1310     logging.info("    Creating the data set for the {0} '{1}'.".
1311                  format(table.get("type", ""), table.get("title", "")))
1312     data = input_data.filter_data(table, continue_on_error=True)
1313
1314     if data is None or data.empty:
1315         logging.warn("    No data for the {0} '{1}'.".
1316                      format(table.get("type", ""), table.get("title", "")))
1317         return
1318
1319     tbl_list = list()
1320     for job, builds in table["data"].items():
1321         for build in builds:
1322             build = str(build)
1323             try:
1324                 version = input_data.metadata(job, build).get("version", "")
1325             except KeyError:
1326                 logging.error("Data for {job}: {build} is not present.".
1327                               format(job=job, build=build))
1328                 return
1329             tbl_list.append(build)
1330             tbl_list.append(version)
1331             for tst_name, tst_data in data[job][build].iteritems():
1332                 if tst_data["status"] != "FAIL":
1333                     continue
1334                 groups = re.search(REGEX_NIC, tst_data["parent"])
1335                 if not groups:
1336                     continue
1337                 nic = groups.group(0)
1338                 tbl_list.append("{0}-{1}".format(nic, tst_data["name"]))
1339
1340     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1341     logging.info("    Writing file: '{0}'".format(file_name))
1342     with open(file_name, "w") as file_handler:
1343         for test in tbl_list:
1344             file_handler.write(test + '\n')
1345
1346
1347 def table_failed_tests(table, input_data):
1348     """Generate the table(s) with algorithm: table_failed_tests
1349     specified in the specification file.
1350
1351     :param table: Table to generate.
1352     :param input_data: Data to process.
1353     :type table: pandas.Series
1354     :type input_data: InputData
1355     """
1356
1357     logging.info("  Generating the table {0} ...".
1358                  format(table.get("title", "")))
1359
1360     # Transform the data
1361     logging.info("    Creating the data set for the {0} '{1}'.".
1362                  format(table.get("type", ""), table.get("title", "")))
1363     data = input_data.filter_data(table, continue_on_error=True)
1364
1365     # Prepare the header of the tables
1366     header = ["Test Case",
1367               "Failures [#]",
1368               "Last Failure [Time]",
1369               "Last Failure [VPP-Build-Id]",
1370               "Last Failure [CSIT-Job-Build-Id]"]
1371
1372     # Generate the data for the table according to the model in the table
1373     # specification
1374
1375     now = dt.utcnow()
1376     timeperiod = timedelta(int(table.get("window", 7)))
1377
1378     tbl_dict = dict()
1379     for job, builds in table["data"].items():
1380         for build in builds:
1381             build = str(build)
1382             for tst_name, tst_data in data[job][build].iteritems():
1383                 if tst_name.lower() in table.get("ignore-list", list()):
1384                     continue
1385                 if tbl_dict.get(tst_name, None) is None:
1386                     groups = re.search(REGEX_NIC, tst_data["parent"])
1387                     if not groups:
1388                         continue
1389                     nic = groups.group(0)
1390                     tbl_dict[tst_name] = {
1391                         "name": "{0}-{1}".format(nic, tst_data["name"]),
1392                         "data": OrderedDict()}
1393                 try:
1394                     generated = input_data.metadata(job, build).\
1395                         get("generated", "")
1396                     if not generated:
1397                         continue
1398                     then = dt.strptime(generated, "%Y%m%d %H:%M")
1399                     if (now - then) <= timeperiod:
1400                         tbl_dict[tst_name]["data"][build] = (
1401                             tst_data["status"],
1402                             generated,
1403                             input_data.metadata(job, build).get("version", ""),
1404                             build)
1405                 except (TypeError, KeyError) as err:
1406                     logging.warning("tst_name: {} - err: {}".
1407                                     format(tst_name, repr(err)))
1408
1409     max_fails = 0
1410     tbl_lst = list()
1411     for tst_data in tbl_dict.values():
1412         fails_nr = 0
1413         for val in tst_data["data"].values():
1414             if val[0] == "FAIL":
1415                 fails_nr += 1
1416                 fails_last_date = val[1]
1417                 fails_last_vpp = val[2]
1418                 fails_last_csit = val[3]
1419         if fails_nr:
1420             max_fails = fails_nr if fails_nr > max_fails else max_fails
1421             tbl_lst.append([tst_data["name"],
1422                             fails_nr,
1423                             fails_last_date,
1424                             fails_last_vpp,
1425                             "mrr-daily-build-{0}".format(fails_last_csit)])
1426
1427     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1428     tbl_sorted = list()
1429     for nrf in range(max_fails, -1, -1):
1430         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1431         tbl_sorted.extend(tbl_fails)
1432     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1433
1434     logging.info("    Writing file: '{0}'".format(file_name))
1435     with open(file_name, "w") as file_handler:
1436         file_handler.write(",".join(header) + "\n")
1437         for test in tbl_sorted:
1438             file_handler.write(",".join([str(item) for item in test]) + '\n')
1439
1440     txt_file_name = "{0}.txt".format(table["output-file"])
1441     logging.info("    Writing file: '{0}'".format(txt_file_name))
1442     convert_csv_to_pretty_txt(file_name, txt_file_name)
1443
1444
1445 def table_failed_tests_html(table, input_data):
1446     """Generate the table(s) with algorithm: table_failed_tests_html
1447     specified in the specification file.
1448
1449     :param table: Table to generate.
1450     :param input_data: Data to process.
1451     :type table: pandas.Series
1452     :type input_data: InputData
1453     """
1454
1455     testbed = table.get("testbed", None)
1456     if testbed is None:
1457         logging.error("The testbed is not defined for the table '{0}'.".
1458                       format(table.get("title", "")))
1459         return
1460
1461     logging.info("  Generating the table {0} ...".
1462                  format(table.get("title", "")))
1463
1464     try:
1465         with open(table["input-file"], 'rb') as csv_file:
1466             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
1467             csv_lst = [item for item in csv_content]
1468     except KeyError:
1469         logging.warning("The input file is not defined.")
1470         return
1471     except csv.Error as err:
1472         logging.warning("Not possible to process the file '{0}'.\n{1}".
1473                         format(table["input-file"], err))
1474         return
1475
1476     # Table:
1477     failed_tests = ET.Element("table", attrib=dict(width="100%", border='0'))
1478
1479     # Table header:
1480     tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor="#7eade7"))
1481     for idx, item in enumerate(csv_lst[0]):
1482         alignment = "left" if idx == 0 else "center"
1483         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
1484         th.text = item
1485
1486     # Rows:
1487     colors = ("#e9f1fb", "#d4e4f7")
1488     for r_idx, row in enumerate(csv_lst[1:]):
1489         background = colors[r_idx % 2]
1490         tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor=background))
1491
1492         # Columns:
1493         for c_idx, item in enumerate(row):
1494             alignment = "left" if c_idx == 0 else "center"
1495             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1496             # Name:
1497             if c_idx == 0:
1498                 url = _generate_url("../trending/", testbed, item)
1499                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1500                 ref.text = item
1501             else:
1502                 td.text = item
1503     try:
1504         with open(table["output-file"], 'w') as html_file:
1505             logging.info("    Writing file: '{0}'".format(table["output-file"]))
1506             html_file.write(".. raw:: html\n\n\t")
1507             html_file.write(ET.tostring(failed_tests))
1508             html_file.write("\n\t<p><br><br></p>\n")
1509     except KeyError:
1510         logging.warning("The output file is not defined.")
1511         return