resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from collections import OrderedDict
  23 from xml.etree import ElementTree as ET
  24 from datetime import datetime as dt
  25 from datetime import timedelta
  26
  27 import plotly.graph_objects as go
  28 import plotly.offline as ploff
  29 import pandas as pd
  30
  31 from numpy import nan, isnan
  32
  33 from pal_utils import mean, stdev, relative_change, classify_anomalies, \
  34     convert_csv_to_pretty_txt, relative_change_stdev
  35
  36
  37 REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*')
  38
  39
  40 def generate_tables(spec, data):
  41     """Generate all tables specified in the specification file.
  42
  43     :param spec: Specification read from the specification file.
  44     :param data: Data to process.
  45     :type spec: Specification
  46     :type data: InputData
  47     """
  48
  49     generator = {
  50         u"table_details": table_details,
  51         u"table_merged_details": table_merged_details,
  52         u"table_perf_comparison": table_perf_comparison,
  53         u"table_perf_comparison_nic": table_perf_comparison_nic,
  54         u"table_nics_comparison": table_nics_comparison,
  55         u"table_soak_vs_ndr": table_soak_vs_ndr,
  56         u"table_perf_trending_dash": table_perf_trending_dash,
  57         u"table_perf_trending_dash_html": table_perf_trending_dash_html,
  58         u"table_last_failed_tests": table_last_failed_tests,
  59         u"table_failed_tests": table_failed_tests,
  60         u"table_failed_tests_html": table_failed_tests_html
  61     }
  62
  63     logging.info(u"Generating the tables ...")
  64     for table in spec.tables:
  65         try:
  66             generator[table[u"algorithm"]](table, data)
  67         except NameError as err:
  68             logging.error(
  69                 f"Probably algorithm {table[u'algorithm']} is not defined: "
  70                 f"{repr(err)}"
  71             )
  72     logging.info(u"Done.")
  73
  74
  75 def table_details(table, input_data):
  76     """Generate the table(s) with algorithm: table_detailed_test_results
  77     specified in the specification file.
  78
  79     :param table: Table to generate.
  80     :param input_data: Data to process.
  81     :type table: pandas.Series
  82     :type input_data: InputData
  83     """
  84
  85     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
  86
  87     # Transform the data
  88     logging.info(
  89         f"    Creating the data set for the {table.get(u'type', u'')} "
  90         f"{table.get(u'title', u'')}."
  91     )
  92     data = input_data.filter_data(table)
  93
  94     # Prepare the header of the tables
  95     header = list()
  96     for column in table[u"columns"]:
  97         header.append(
  98             u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
  99         )
 100
 101     # Generate the data for the table according to the model in the table
 102     # specification
 103     job = list(table[u"data"].keys())[0]
 104     build = str(table[u"data"][job][0])
 105     try:
 106         suites = input_data.suites(job, build)
 107     except KeyError:
 108         logging.error(
 109             u"    No data available. The table will not be generated."
 110         )
 111         return
 112
 113     for suite in suites.values:
 114         # Generate data
 115         suite_name = suite[u"name"]
 116         table_lst = list()
 117         for test in data[job][build].keys():
 118             if data[job][build][test][u"parent"] not in suite_name:
 119                 continue
 120             row_lst = list()
 121             for column in table[u"columns"]:
 122                 try:
 123                     col_data = str(data[job][build][test][column[
 124                         u"data"].split(" ")[1]]).replace(u'"', u'""')
 125                     if column[u"data"].split(u" ")[1] in \
 126                         (u"conf-history", u"show-run"):
 127                         col_data = col_data.replace(u" |br| ", u"", 1)
 128                         col_data = f" |prein| {col_data[:-5]} |preout| "
 129                     row_lst.append(f'"{col_data}"')
 130                 except KeyError:
 131                     row_lst.append(u"No data")
 132             table_lst.append(row_lst)
 133
 134         # Write the data to file
 135         if table_lst:
 136             file_name = (
 137                 f"{table[u'output-file']}_{suite_name}"
 138                 f"{table[u'output-file-ext']}"
 139             )
 140             logging.info(f"      Writing file: {file_name}")
 141             with open(file_name, u"wt") as file_handler:
 142                 file_handler.write(u",".join(header) + u"\n")
 143                 for item in table_lst:
 144                     file_handler.write(u",".join(item) + u"\n")
 145
 146     logging.info(u"  Done.")
 147
 148
 149 def table_merged_details(table, input_data):
 150     """Generate the table(s) with algorithm: table_merged_details
 151     specified in the specification file.
 152
 153     :param table: Table to generate.
 154     :param input_data: Data to process.
 155     :type table: pandas.Series
 156     :type input_data: InputData
 157     """
 158
 159     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 160     # Transform the data
 161     logging.info(
 162         f"    Creating the data set for the {table.get(u'type', u'')} "
 163         f"{table.get(u'title', u'')}."
 164     )
 165     data = input_data.filter_data(table, continue_on_error=True)
 166     data = input_data.merge_data(data)
 167     data.sort_index(inplace=True)
 168
 169     logging.info(
 170         f"    Creating the data set for the {table.get(u'type', u'')} "
 171         f"{table.get(u'title', u'')}."
 172     )
 173     suites = input_data.filter_data(
 174         table, continue_on_error=True, data_set=u"suites")
 175     suites = input_data.merge_data(suites)
 176
 177     # Prepare the header of the tables
 178     header = list()
 179     for column in table[u"columns"]:
 180         header.append(
 181             u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
 182         )
 183
 184     for suite in suites.values:
 185         # Generate data
 186         suite_name = suite[u"name"]
 187         table_lst = list()
 188         for test in data.keys():
 189             if data[test][u"parent"] not in suite_name:
 190                 continue
 191             row_lst = list()
 192             for column in table[u"columns"]:
 193                 try:
 194                     col_data = str(data[test][column[
 195                         u"data"].split(u" ")[1]]).replace(u'"', u'""')
 196                     col_data = col_data.replace(
 197                         u"No Data", u"Not Captured     "
 198                     )
 199                     if column[u"data"].split(u" ")[1] in \
 200                         (u"conf-history", u"show-run"):
 201                         col_data = col_data.replace(u" |br| ", u"", 1)
 202                         col_data = f" |prein| {col_data[:-5]} |preout| "
 203                     row_lst.append(f'"{col_data}"')
 204                 except KeyError:
 205                     row_lst.append(u'"Not captured"')
 206             table_lst.append(row_lst)
 207
 208         # Write the data to file
 209         if table_lst:
 210             file_name = (
 211                 f"{table[u'output-file']}_{suite_name}"
 212                 f"{table[u'output-file-ext']}"
 213             )
 214             logging.info(f"      Writing file: {file_name}")
 215             with open(file_name, u"wt") as file_handler:
 216                 file_handler.write(u",".join(header) + u"\n")
 217                 for item in table_lst:
 218                     file_handler.write(u",".join(item) + u"\n")
 219
 220     logging.info(u"  Done.")
 221
 222
 223 def _tpc_modify_test_name(test_name):
 224     """Modify a test name by replacing its parts.
 225
 226     :param test_name: Test name to be modified.
 227     :type test_name: str
 228     :returns: Modified test name.
 229     :rtype: str
 230     """
 231     test_name_mod = test_name.\
 232         replace(u"-ndrpdrdisc", u""). \
 233         replace(u"-ndrpdr", u"").\
 234         replace(u"-pdrdisc", u""). \
 235         replace(u"-ndrdisc", u"").\
 236         replace(u"-pdr", u""). \
 237         replace(u"-ndr", u""). \
 238         replace(u"1t1c", u"1c").\
 239         replace(u"2t1c", u"1c"). \
 240         replace(u"2t2c", u"2c").\
 241         replace(u"4t2c", u"2c"). \
 242         replace(u"4t4c", u"4c").\
 243         replace(u"8t4c", u"4c")
 244
 245     return re.sub(REGEX_NIC, u"", test_name_mod)
 246
 247
 248 def _tpc_modify_displayed_test_name(test_name):
 249     """Modify a test name which is displayed in a table by replacing its parts.
 250
 251     :param test_name: Test name to be modified.
 252     :type test_name: str
 253     :returns: Modified test name.
 254     :rtype: str
 255     """
 256     return test_name.\
 257         replace(u"1t1c", u"1c").\
 258         replace(u"2t1c", u"1c"). \
 259         replace(u"2t2c", u"2c").\
 260         replace(u"4t2c", u"2c"). \
 261         replace(u"4t4c", u"4c").\
 262         replace(u"8t4c", u"4c")
 263
 264
 265 def _tpc_insert_data(target, src, include_tests):
 266     """Insert src data to the target structure.
 267
 268     :param target: Target structure where the data is placed.
 269     :param src: Source data to be placed into the target stucture.
 270     :param include_tests: Which results will be included (MRR, NDR, PDR).
 271     :type target: list
 272     :type src: dict
 273     :type include_tests: str
 274     """
 275     try:
 276         if include_tests == u"MRR":
 277             target.append(src[u"result"][u"receive-rate"])
 278         elif include_tests == u"PDR":
 279             target.append(src[u"throughput"][u"PDR"][u"LOWER"])
 280         elif include_tests == u"NDR":
 281             target.append(src[u"throughput"][u"NDR"][u"LOWER"])
 282     except (KeyError, TypeError):
 283         pass
 284
 285
 286 def _tpc_sort_table(table):
 287     """Sort the table this way:
 288
 289     1. Put "New in CSIT-XXXX" at the first place.
 290     2. Put "See footnote" at the second place.
 291     3. Sort the rest by "Delta".
 292
 293     :param table: Table to sort.
 294     :type table: list
 295     :returns: Sorted table.
 296     :rtype: list
 297     """
 298
 299
 300     tbl_new = list()
 301     tbl_see = list()
 302     tbl_delta = list()
 303     for item in table:
 304         if isinstance(item[-1], str):
 305             if u"New in CSIT" in item[-1]:
 306                 tbl_new.append(item)
 307             elif u"See footnote" in item[-1]:
 308                 tbl_see.append(item)
 309         else:
 310             tbl_delta.append(item)
 311
 312     # Sort the tables:
 313     tbl_new.sort(key=lambda rel: rel[0], reverse=False)
 314     tbl_see.sort(key=lambda rel: rel[0], reverse=False)
 315     tbl_see.sort(key=lambda rel: rel[-1], reverse=False)
 316     tbl_delta.sort(key=lambda rel: rel[-1], reverse=True)
 317
 318     # Put the tables together:
 319     table = list()
 320     table.extend(tbl_new)
 321     table.extend(tbl_see)
 322     table.extend(tbl_delta)
 323
 324     return table
 325
 326
 327 def _tpc_generate_html_table(header, data, output_file_name):
 328     """Generate html table from input data with simple sorting possibility.
 329
 330     :param header: Table header.
 331     :param data: Input data to be included in the table. It is a list of lists.
 332         Inner lists are rows in the table. All inner lists must be of the same
 333         length. The length of these lists must be the same as the length of the
 334         header.
 335     :param output_file_name: The name (relative or full path) where the
 336         generated html table is written.
 337     :type header: list
 338     :type data: list of lists
 339     :type output_file_name: str
 340     """
 341
 342     df_data = pd.DataFrame(data, columns=header)
 343
 344     df_sorted = [df_data.sort_values(
 345         by=[key, header[0]], ascending=[True, True]
 346         if key != header[0] else [False, True]) for key in header]
 347     df_sorted_rev = [df_data.sort_values(
 348         by=[key, header[0]], ascending=[False, True]
 349         if key != header[0] else [True, True]) for key in header]
 350     df_sorted.extend(df_sorted_rev)
 351
 352     fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
 353                    for idx in range(len(df_data))]]
 354     table_header = dict(
 355         values=[f"<b>{item}</b>" for item in header],
 356         fill_color=u"#7eade7",
 357         align=[u"left", u"center"]
 358     )
 359
 360     fig = go.Figure()
 361
 362     for table in df_sorted:
 363         columns = [table.get(col) for col in header]
 364         fig.add_trace(
 365             go.Table(
 366                 columnwidth=[30, 10],
 367                 header=table_header,
 368                 cells=dict(
 369                     values=columns,
 370                     fill_color=fill_color,
 371                     align=[u"left", u"right"]
 372                 )
 373             )
 374         )
 375
 376     buttons = list()
 377     menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
 378     menu_items_rev = [f"<b>{itm}</b> (descending)" for itm in header]
 379     menu_items.extend(menu_items_rev)
 380     for idx, hdr in enumerate(menu_items):
 381         visible = [False, ] * len(menu_items)
 382         visible[idx] = True
 383         buttons.append(
 384             dict(
 385                 label=hdr.replace(u" [Mpps]", u""),
 386                 method=u"update",
 387                 args=[{u"visible": visible}],
 388             )
 389         )
 390
 391     fig.update_layout(
 392         updatemenus=[
 393             go.layout.Updatemenu(
 394                 type=u"dropdown",
 395                 direction=u"down",
 396                 x=0.03,
 397                 xanchor=u"left",
 398                 y=1.045,
 399                 yanchor=u"top",
 400                 active=len(menu_items) - 1,
 401                 buttons=list(buttons)
 402             )
 403         ],
 404         annotations=[
 405             go.layout.Annotation(
 406                 text=u"<b>Sort by:</b>",
 407                 x=0,
 408                 xref=u"paper",
 409                 y=1.035,
 410                 yref=u"paper",
 411                 align=u"left",
 412                 showarrow=False
 413             )
 414         ]
 415     )
 416
 417     ploff.plot(fig, show_link=False, auto_open=False, filename=output_file_name)
 418
 419
 420 def table_perf_comparison(table, input_data):
 421     """Generate the table(s) with algorithm: table_perf_comparison
 422     specified in the specification file.
 423
 424     :param table: Table to generate.
 425     :param input_data: Data to process.
 426     :type table: pandas.Series
 427     :type input_data: InputData
 428     """
 429
 430     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 431
 432     # Transform the data
 433     logging.info(
 434         f"    Creating the data set for the {table.get(u'type', u'')} "
 435         f"{table.get(u'title', u'')}."
 436     )
 437     data = input_data.filter_data(table, continue_on_error=True)
 438
 439     # Prepare the header of the tables
 440     try:
 441         header = [u"Test case", ]
 442
 443         if table[u"include-tests"] == u"MRR":
 444             hdr_param = u"Rec Rate"
 445         else:
 446             hdr_param = u"Thput"
 447
 448         history = table.get(u"history", list())
 449         for item in history:
 450             header.extend(
 451                 [
 452                     f"{item[u'title']} {hdr_param} [Mpps]",
 453                     f"{item[u'title']} Stdev [Mpps]"
 454                 ]
 455             )
 456         header.extend(
 457             [
 458                 f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
 459                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
 460                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
 461                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
 462                 u"Delta [%]"
 463             ]
 464         )
 465         header_str = u",".join(header) + u"\n"
 466     except (AttributeError, KeyError) as err:
 467         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 468         return
 469
 470     # Prepare data to the table:
 471     tbl_dict = dict()
 472     # topo = ""
 473     for job, builds in table[u"reference"][u"data"].items():
 474         # topo = u"2n-skx" if u"2n-skx" in job else u""
 475         for build in builds:
 476             for tst_name, tst_data in data[job][str(build)].items():
 477                 tst_name_mod = _tpc_modify_test_name(tst_name)
 478                 if u"across topologies" in table[u"title"].lower():
 479                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 480                 if tbl_dict.get(tst_name_mod, None) is None:
 481                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
 482                     nic = groups.group(0) if groups else u""
 483                     name = \
 484                         f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 485                     if u"across testbeds" in table[u"title"].lower() or \
 486                             u"across topologies" in table[u"title"].lower():
 487                         name = _tpc_modify_displayed_test_name(name)
 488                     tbl_dict[tst_name_mod] = {
 489                         u"name": name,
 490                         u"ref-data": list(),
 491                         u"cmp-data": list()
 492                     }
 493                 _tpc_insert_data(target=tbl_dict[tst_name_mod][u"ref-data"],
 494                                  src=tst_data,
 495                                  include_tests=table[u"include-tests"])
 496
 497     replacement = table[u"reference"].get(u"data-replacement", None)
 498     if replacement:
 499         create_new_list = True
 500         rpl_data = input_data.filter_data(
 501             table, data=replacement, continue_on_error=True)
 502         for job, builds in replacement.items():
 503             for build in builds:
 504                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 505                     tst_name_mod = _tpc_modify_test_name(tst_name)
 506                     if u"across topologies" in table[u"title"].lower():
 507                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 508                     if tbl_dict.get(tst_name_mod, None) is None:
 509                         name = \
 510                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 511                         if u"across testbeds" in table[u"title"].lower() or \
 512                                 u"across topologies" in table[u"title"].lower():
 513                             name = _tpc_modify_displayed_test_name(name)
 514                         tbl_dict[tst_name_mod] = {
 515                             u"name": name,
 516                             u"ref-data": list(),
 517                             u"cmp-data": list()
 518                         }
 519                     if create_new_list:
 520                         create_new_list = False
 521                         tbl_dict[tst_name_mod][u"ref-data"] = list()
 522
 523                     _tpc_insert_data(
 524                         target=tbl_dict[tst_name_mod][u"ref-data"],
 525                         src=tst_data,
 526                         include_tests=table[u"include-tests"]
 527                     )
 528
 529     for job, builds in table[u"compare"][u"data"].items():
 530         for build in builds:
 531             for tst_name, tst_data in data[job][str(build)].items():
 532                 tst_name_mod = _tpc_modify_test_name(tst_name)
 533                 if u"across topologies" in table[u"title"].lower():
 534                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 535                 if tbl_dict.get(tst_name_mod, None) is None:
 536                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
 537                     nic = groups.group(0) if groups else u""
 538                     name = \
 539                         f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 540                     if u"across testbeds" in table[u"title"].lower() or \
 541                             u"across topologies" in table[u"title"].lower():
 542                         name = _tpc_modify_displayed_test_name(name)
 543                     tbl_dict[tst_name_mod] = {
 544                         u"name": name,
 545                         u"ref-data": list(),
 546                         u"cmp-data": list()
 547                     }
 548                 _tpc_insert_data(
 549                     target=tbl_dict[tst_name_mod][u"cmp-data"],
 550                     src=tst_data,
 551                     include_tests=table[u"include-tests"]
 552                 )
 553
 554     replacement = table[u"compare"].get(u"data-replacement", None)
 555     if replacement:
 556         create_new_list = True
 557         rpl_data = input_data.filter_data(
 558             table, data=replacement, continue_on_error=True)
 559         for job, builds in replacement.items():
 560             for build in builds:
 561                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 562                     tst_name_mod = _tpc_modify_test_name(tst_name)
 563                     if u"across topologies" in table[u"title"].lower():
 564                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 565                     if tbl_dict.get(tst_name_mod, None) is None:
 566                         name = \
 567                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 568                         if u"across testbeds" in table[u"title"].lower() or \
 569                                 u"across topologies" in table[u"title"].lower():
 570                             name = _tpc_modify_displayed_test_name(name)
 571                         tbl_dict[tst_name_mod] = {
 572                             u"name": name,
 573                             u"ref-data": list(),
 574                             u"cmp-data": list()
 575                         }
 576                     if create_new_list:
 577                         create_new_list = False
 578                         tbl_dict[tst_name_mod][u"cmp-data"] = list()
 579
 580                     _tpc_insert_data(
 581                         target=tbl_dict[tst_name_mod][u"cmp-data"],
 582                         src=tst_data,
 583                         include_tests=table[u"include-tests"]
 584                     )
 585
 586     for item in history:
 587         for job, builds in item[u"data"].items():
 588             for build in builds:
 589                 for tst_name, tst_data in data[job][str(build)].items():
 590                     tst_name_mod = _tpc_modify_test_name(tst_name)
 591                     if u"across topologies" in table[u"title"].lower():
 592                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 593                     if tbl_dict.get(tst_name_mod, None) is None:
 594                         continue
 595                     if tbl_dict[tst_name_mod].get(u"history", None) is None:
 596                         tbl_dict[tst_name_mod][u"history"] = OrderedDict()
 597                     if tbl_dict[tst_name_mod][u"history"].\
 598                             get(item[u"title"], None) is None:
 599                         tbl_dict[tst_name_mod][u"history"][item[
 600                             u"title"]] = list()
 601                     try:
 602                         if table[u"include-tests"] == u"MRR":
 603                             res = tst_data[u"result"][u"receive-rate"]
 604                         elif table[u"include-tests"] == u"PDR":
 605                             res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
 606                         elif table[u"include-tests"] == u"NDR":
 607                             res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
 608                         else:
 609                             continue
 610                         tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
 611                             append(res)
 612                     except (TypeError, KeyError):
 613                         pass
 614
 615     tbl_lst = list()
 616     footnote = False
 617     for tst_name in tbl_dict:
 618         item = [tbl_dict[tst_name][u"name"], ]
 619         if history:
 620             if tbl_dict[tst_name].get(u"history", None) is not None:
 621                 for hist_data in tbl_dict[tst_name][u"history"].values():
 622                     if hist_data:
 623                         item.append(round(mean(hist_data) / 1000000, 2))
 624                         item.append(round(stdev(hist_data) / 1000000, 2))
 625                     else:
 626                         item.extend([u"Not tested", u"Not tested"])
 627             else:
 628                 item.extend([u"Not tested", u"Not tested"])
 629         data_t = tbl_dict[tst_name][u"ref-data"]
 630         if data_t:
 631             item.append(round(mean(data_t) / 1000000, 2))
 632             item.append(round(stdev(data_t) / 1000000, 2))
 633         else:
 634             item.extend([u"Not tested", u"Not tested"])
 635         data_t = tbl_dict[tst_name][u"cmp-data"]
 636         if data_t:
 637             item.append(round(mean(data_t) / 1000000, 2))
 638             item.append(round(stdev(data_t) / 1000000, 2))
 639         else:
 640             item.extend([u"Not tested", u"Not tested"])
 641         if item[-2] == u"Not tested":
 642             pass
 643         elif item[-4] == u"Not tested":
 644             item.append(u"New in CSIT-2001")
 645         # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
 646         #     item.append(u"See footnote [1]")
 647         #     footnote = True
 648         elif item[-4] != 0:
 649             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 650         if (len(item) == len(header)) and (item[-3] != u"Not tested"):
 651             tbl_lst.append(item)
 652
 653     tbl_lst = _tpc_sort_table(tbl_lst)
 654
 655     # Generate csv tables:
 656     csv_file = f"{table[u'output-file']}.csv"
 657     with open(csv_file, u"wt") as file_handler:
 658         file_handler.write(header_str)
 659         for test in tbl_lst:
 660             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
 661
 662     txt_file_name = f"{table[u'output-file']}.txt"
 663     convert_csv_to_pretty_txt(csv_file, txt_file_name)
 664
 665     if footnote:
 666         with open(txt_file_name, u'a') as txt_file:
 667             txt_file.writelines([
 668                 u"\nFootnotes:\n",
 669                 u"[1] CSIT-1908 changed test methodology of dot1q tests in "
 670                 u"2-node testbeds, dot1q encapsulation is now used on both "
 671                 u"links of SUT.\n",
 672                 u"    Previously dot1q was used only on a single link with the "
 673                 u"other link carrying untagged Ethernet frames. This changes "
 674                 u"results\n",
 675                 u"    in slightly lower throughput in CSIT-1908 for these "
 676                 u"tests. See release notes."
 677             ])
 678
 679     # Generate html table:
 680     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
 681
 682
 683 def table_perf_comparison_nic(table, input_data):
 684     """Generate the table(s) with algorithm: table_perf_comparison
 685     specified in the specification file.
 686
 687     :param table: Table to generate.
 688     :param input_data: Data to process.
 689     :type table: pandas.Series
 690     :type input_data: InputData
 691     """
 692
 693     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 694
 695     # Transform the data
 696     logging.info(
 697         f"    Creating the data set for the {table.get(u'type', u'')} "
 698         f"{table.get(u'title', u'')}."
 699     )
 700     data = input_data.filter_data(table, continue_on_error=True)
 701
 702     # Prepare the header of the tables
 703     try:
 704         header = [u"Test case", ]
 705
 706         if table[u"include-tests"] == u"MRR":
 707             hdr_param = u"Rec Rate"
 708         else:
 709             hdr_param = u"Thput"
 710
 711         history = table.get(u"history", list())
 712         for item in history:
 713             header.extend(
 714                 [
 715                     f"{item[u'title']} {hdr_param} [Mpps]",
 716                     f"{item[u'title']} Stdev [Mpps]"
 717                 ]
 718             )
 719         header.extend(
 720             [
 721                 f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
 722                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
 723                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
 724                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
 725                 u"Delta [%]"
 726             ]
 727         )
 728         header_str = u",".join(header) + u"\n"
 729     except (AttributeError, KeyError) as err:
 730         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 731         return
 732
 733     # Prepare data to the table:
 734     tbl_dict = dict()
 735     # topo = u""
 736     for job, builds in table[u"reference"][u"data"].items():
 737         # topo = u"2n-skx" if u"2n-skx" in job else u""
 738         for build in builds:
 739             for tst_name, tst_data in data[job][str(build)].items():
 740                 if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
 741                     continue
 742                 tst_name_mod = _tpc_modify_test_name(tst_name)
 743                 if u"across topologies" in table[u"title"].lower():
 744                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 745                 if tbl_dict.get(tst_name_mod, None) is None:
 746                     name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 747                     if u"across testbeds" in table[u"title"].lower() or \
 748                             u"across topologies" in table[u"title"].lower():
 749                         name = _tpc_modify_displayed_test_name(name)
 750                     tbl_dict[tst_name_mod] = {
 751                         u"name": name,
 752                         u"ref-data": list(),
 753                         u"cmp-data": list()
 754                     }
 755                 _tpc_insert_data(
 756                     target=tbl_dict[tst_name_mod][u"ref-data"],
 757                     src=tst_data,
 758                     include_tests=table[u"include-tests"]
 759                 )
 760
 761     replacement = table[u"reference"].get(u"data-replacement", None)
 762     if replacement:
 763         create_new_list = True
 764         rpl_data = input_data.filter_data(
 765             table, data=replacement, continue_on_error=True)
 766         for job, builds in replacement.items():
 767             for build in builds:
 768                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 769                     if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
 770                         continue
 771                     tst_name_mod = _tpc_modify_test_name(tst_name)
 772                     if u"across topologies" in table[u"title"].lower():
 773                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 774                     if tbl_dict.get(tst_name_mod, None) is None:
 775                         name = \
 776                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 777                         if u"across testbeds" in table[u"title"].lower() or \
 778                                 u"across topologies" in table[u"title"].lower():
 779                             name = _tpc_modify_displayed_test_name(name)
 780                         tbl_dict[tst_name_mod] = {
 781                             u"name": name,
 782                             u"ref-data": list(),
 783                             u"cmp-data": list()
 784                         }
 785                     if create_new_list:
 786                         create_new_list = False
 787                         tbl_dict[tst_name_mod][u"ref-data"] = list()
 788
 789                     _tpc_insert_data(
 790                         target=tbl_dict[tst_name_mod][u"ref-data"],
 791                         src=tst_data,
 792                         include_tests=table[u"include-tests"]
 793                     )
 794
 795     for job, builds in table[u"compare"][u"data"].items():
 796         for build in builds:
 797             for tst_name, tst_data in data[job][str(build)].items():
 798                 if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
 799                     continue
 800                 tst_name_mod = _tpc_modify_test_name(tst_name)
 801                 if u"across topologies" in table[u"title"].lower():
 802                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 803                 if tbl_dict.get(tst_name_mod, None) is None:
 804                     name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 805                     if u"across testbeds" in table[u"title"].lower() or \
 806                             u"across topologies" in table[u"title"].lower():
 807                         name = _tpc_modify_displayed_test_name(name)
 808                     tbl_dict[tst_name_mod] = {
 809                         u"name": name,
 810                         u"ref-data": list(),
 811                         u"cmp-data": list()
 812                     }
 813                 _tpc_insert_data(
 814                     target=tbl_dict[tst_name_mod][u"cmp-data"],
 815                     src=tst_data,
 816                     include_tests=table[u"include-tests"]
 817                 )
 818
 819     replacement = table[u"compare"].get(u"data-replacement", None)
 820     if replacement:
 821         create_new_list = True
 822         rpl_data = input_data.filter_data(
 823             table, data=replacement, continue_on_error=True)
 824         for job, builds in replacement.items():
 825             for build in builds:
 826                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 827                     if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
 828                         continue
 829                     tst_name_mod = _tpc_modify_test_name(tst_name)
 830                     if u"across topologies" in table[u"title"].lower():
 831                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 832                     if tbl_dict.get(tst_name_mod, None) is None:
 833                         name = \
 834                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 835                         if u"across testbeds" in table[u"title"].lower() or \
 836                                 u"across topologies" in table[u"title"].lower():
 837                             name = _tpc_modify_displayed_test_name(name)
 838                         tbl_dict[tst_name_mod] = {
 839                             u"name": name,
 840                             u"ref-data": list(),
 841                             u"cmp-data": list()
 842                         }
 843                     if create_new_list:
 844                         create_new_list = False
 845                         tbl_dict[tst_name_mod][u"cmp-data"] = list()
 846
 847                     _tpc_insert_data(
 848                         target=tbl_dict[tst_name_mod][u"cmp-data"],
 849                         src=tst_data,
 850                         include_tests=table[u"include-tests"]
 851                     )
 852
 853     for item in history:
 854         for job, builds in item[u"data"].items():
 855             for build in builds:
 856                 for tst_name, tst_data in data[job][str(build)].items():
 857                     if item[u"nic"] not in tst_data[u"tags"]:
 858                         continue
 859                     tst_name_mod = _tpc_modify_test_name(tst_name)
 860                     if u"across topologies" in table[u"title"].lower():
 861                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 862                     if tbl_dict.get(tst_name_mod, None) is None:
 863                         continue
 864                     if tbl_dict[tst_name_mod].get(u"history", None) is None:
 865                         tbl_dict[tst_name_mod][u"history"] = OrderedDict()
 866                     if tbl_dict[tst_name_mod][u"history"].\
 867                             get(item[u"title"], None) is None:
 868                         tbl_dict[tst_name_mod][u"history"][item[
 869                             u"title"]] = list()
 870                     try:
 871                         if table[u"include-tests"] == u"MRR":
 872                             res = tst_data[u"result"][u"receive-rate"]
 873                         elif table[u"include-tests"] == u"PDR":
 874                             res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
 875                         elif table[u"include-tests"] == u"NDR":
 876                             res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
 877                         else:
 878                             continue
 879                         tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
 880                             append(res)
 881                     except (TypeError, KeyError):
 882                         pass
 883
 884     tbl_lst = list()
 885     footnote = False
 886     for tst_name in tbl_dict:
 887         item = [tbl_dict[tst_name][u"name"], ]
 888         if history:
 889             if tbl_dict[tst_name].get(u"history", None) is not None:
 890                 for hist_data in tbl_dict[tst_name][u"history"].values():
 891                     if hist_data:
 892                         item.append(round(mean(hist_data) / 1000000, 2))
 893                         item.append(round(stdev(hist_data) / 1000000, 2))
 894                     else:
 895                         item.extend([u"Not tested", u"Not tested"])
 896             else:
 897                 item.extend([u"Not tested", u"Not tested"])
 898         data_t = tbl_dict[tst_name][u"ref-data"]
 899         if data_t:
 900             item.append(round(mean(data_t) / 1000000, 2))
 901             item.append(round(stdev(data_t) / 1000000, 2))
 902         else:
 903             item.extend([u"Not tested", u"Not tested"])
 904         data_t = tbl_dict[tst_name][u"cmp-data"]
 905         if data_t:
 906             item.append(round(mean(data_t) / 1000000, 2))
 907             item.append(round(stdev(data_t) / 1000000, 2))
 908         else:
 909             item.extend([u"Not tested", u"Not tested"])
 910         if item[-2] == u"Not tested":
 911             pass
 912         elif item[-4] == u"Not tested":
 913             item.append(u"New in CSIT-2001")
 914         # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
 915         #     item.append(u"See footnote [1]")
 916         #     footnote = True
 917         elif item[-4] != 0:
 918             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 919         if (len(item) == len(header)) and (item[-3] != u"Not tested"):
 920             tbl_lst.append(item)
 921
 922     tbl_lst = _tpc_sort_table(tbl_lst)
 923
 924     # Generate csv tables:
 925     csv_file = f"{table[u'output-file']}.csv"
 926     with open(csv_file, u"wt") as file_handler:
 927         file_handler.write(header_str)
 928         for test in tbl_lst:
 929             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
 930
 931     txt_file_name = f"{table[u'output-file']}.txt"
 932     convert_csv_to_pretty_txt(csv_file, txt_file_name)
 933
 934     if footnote:
 935         with open(txt_file_name, u'a') as txt_file:
 936             txt_file.writelines([
 937                 u"\nFootnotes:\n",
 938                 u"[1] CSIT-1908 changed test methodology of dot1q tests in "
 939                 u"2-node testbeds, dot1q encapsulation is now used on both "
 940                 u"links of SUT.\n",
 941                 u"    Previously dot1q was used only on a single link with the "
 942                 u"other link carrying untagged Ethernet frames. This changes "
 943                 u"results\n",
 944                 u"    in slightly lower throughput in CSIT-1908 for these "
 945                 u"tests. See release notes."
 946             ])
 947
 948     # Generate html table:
 949     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
 950
 951
 952 def table_nics_comparison(table, input_data):
 953     """Generate the table(s) with algorithm: table_nics_comparison
 954     specified in the specification file.
 955
 956     :param table: Table to generate.
 957     :param input_data: Data to process.
 958     :type table: pandas.Series
 959     :type input_data: InputData
 960     """
 961
 962     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 963
 964     # Transform the data
 965     logging.info(
 966         f"    Creating the data set for the {table.get(u'type', u'')} "
 967         f"{table.get(u'title', u'')}."
 968     )
 969     data = input_data.filter_data(table, continue_on_error=True)
 970
 971     # Prepare the header of the tables
 972     try:
 973         header = [u"Test case", ]
 974
 975         if table[u"include-tests"] == u"MRR":
 976             hdr_param = u"Rec Rate"
 977         else:
 978             hdr_param = u"Thput"
 979
 980         header.extend(
 981             [
 982                 f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
 983                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
 984                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
 985                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
 986                 u"Delta [%]"
 987             ]
 988         )
 989
 990     except (AttributeError, KeyError) as err:
 991         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 992         return
 993
 994     # Prepare data to the table:
 995     tbl_dict = dict()
 996     for job, builds in table[u"data"].items():
 997         for build in builds:
 998             for tst_name, tst_data in data[job][str(build)].items():
 999                 tst_name_mod = _tpc_modify_test_name(tst_name)
1000                 if tbl_dict.get(tst_name_mod, None) is None:
1001                     name = u"-".join(tst_data[u"name"].split(u"-")[:-1])
1002                     tbl_dict[tst_name_mod] = {
1003                         u"name": name,
1004                         u"ref-data": list(),
1005                         u"cmp-data": list()
1006                     }
1007                 try:
1008                     result = None
1009                     if table[u"include-tests"] == u"MRR":
1010                         result = tst_data[u"result"][u"receive-rate"]
1011                     elif table[u"include-tests"] == u"PDR":
1012                         result = tst_data[u"throughput"][u"PDR"][u"LOWER"]
1013                     elif table[u"include-tests"] == u"NDR":
1014                         result = tst_data[u"throughput"][u"NDR"][u"LOWER"]
1015                     else:
1016                         continue
1017
1018                     if result and \
1019                             table[u"reference"][u"nic"] in tst_data[u"tags"]:
1020                         tbl_dict[tst_name_mod][u"ref-data"].append(result)
1021                     elif result and \
1022                             table[u"compare"][u"nic"] in tst_data[u"tags"]:
1023                         tbl_dict[tst_name_mod][u"cmp-data"].append(result)
1024                 except (TypeError, KeyError) as err:
1025                     logging.debug(f"No data for {tst_name}\n{repr(err)}")
1026                     # No data in output.xml for this test
1027
1028     tbl_lst = list()
1029     for tst_name in tbl_dict:
1030         item = [tbl_dict[tst_name][u"name"], ]
1031         data_t = tbl_dict[tst_name][u"ref-data"]
1032         if data_t:
1033             item.append(round(mean(data_t) / 1000000, 2))
1034             item.append(round(stdev(data_t) / 1000000, 2))
1035         else:
1036             item.extend([None, None])
1037         data_t = tbl_dict[tst_name][u"cmp-data"]
1038         if data_t:
1039             item.append(round(mean(data_t) / 1000000, 2))
1040             item.append(round(stdev(data_t) / 1000000, 2))
1041         else:
1042             item.extend([None, None])
1043         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
1044             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
1045         if len(item) == len(header):
1046             tbl_lst.append(item)
1047
1048     # Sort the table according to the relative change
1049     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1050
1051     # Generate csv tables:
1052     with open(f"{table[u'output-file']}.csv", u"wt") as file_handler:
1053         file_handler.write(u",".join(header) + u"\n")
1054         for test in tbl_lst:
1055             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
1056
1057     convert_csv_to_pretty_txt(f"{table[u'output-file']}.csv",
1058                               f"{table[u'output-file']}.txt")
1059
1060     # Generate html table:
1061     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
1062
1063
1064 def table_soak_vs_ndr(table, input_data):
1065     """Generate the table(s) with algorithm: table_soak_vs_ndr
1066     specified in the specification file.
1067
1068     :param table: Table to generate.
1069     :param input_data: Data to process.
1070     :type table: pandas.Series
1071     :type input_data: InputData
1072     """
1073
1074     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1075
1076     # Transform the data
1077     logging.info(
1078         f"    Creating the data set for the {table.get(u'type', u'')} "
1079         f"{table.get(u'title', u'')}."
1080     )
1081     data = input_data.filter_data(table, continue_on_error=True)
1082
1083     # Prepare the header of the table
1084     try:
1085         header = [
1086             u"Test case",
1087             f"{table[u'reference'][u'title']} Thput [Mpps]",
1088             f"{table[u'reference'][u'title']} Stdev [Mpps]",
1089             f"{table[u'compare'][u'title']} Thput [Mpps]",
1090             f"{table[u'compare'][u'title']} Stdev [Mpps]",
1091             u"Delta [%]", u"Stdev of delta [%]"
1092         ]
1093         header_str = u",".join(header) + u"\n"
1094     except (AttributeError, KeyError) as err:
1095         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1096         return
1097
1098     # Create a list of available SOAK test results:
1099     tbl_dict = dict()
1100     for job, builds in table[u"compare"][u"data"].items():
1101         for build in builds:
1102             for tst_name, tst_data in data[job][str(build)].items():
1103                 if tst_data[u"type"] == u"SOAK":
1104                     tst_name_mod = tst_name.replace(u"-soak", u"")
1105                     if tbl_dict.get(tst_name_mod, None) is None:
1106                         groups = re.search(REGEX_NIC, tst_data[u"parent"])
1107                         nic = groups.group(0) if groups else u""
1108                         name = (
1109                             f"{nic}-"
1110                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
1111                         )
1112                         tbl_dict[tst_name_mod] = {
1113                             u"name": name,
1114                             u"ref-data": list(),
1115                             u"cmp-data": list()
1116                         }
1117                     try:
1118                         tbl_dict[tst_name_mod][u"cmp-data"].append(
1119                             tst_data[u"throughput"][u"LOWER"])
1120                     except (KeyError, TypeError):
1121                         pass
1122     tests_lst = tbl_dict.keys()
1123
1124     # Add corresponding NDR test results:
1125     for job, builds in table[u"reference"][u"data"].items():
1126         for build in builds:
1127             for tst_name, tst_data in data[job][str(build)].items():
1128                 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
1129                     replace(u"-mrr", u"")
1130                 if tst_name_mod not in tests_lst:
1131                     continue
1132                 try:
1133                     if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
1134                         continue
1135                     if table[u"include-tests"] == u"MRR":
1136                         result = tst_data[u"result"][u"receive-rate"]
1137                     elif table[u"include-tests"] == u"PDR":
1138                         result = \
1139                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
1140                     elif table[u"include-tests"] == u"NDR":
1141                         result = \
1142                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
1143                     else:
1144                         result = None
1145                     if result is not None:
1146                         tbl_dict[tst_name_mod][u"ref-data"].append(
1147                             result)
1148                 except (KeyError, TypeError):
1149                     continue
1150
1151     tbl_lst = list()
1152     for tst_name in tbl_dict:
1153         item = [tbl_dict[tst_name][u"name"], ]
1154         data_r = tbl_dict[tst_name][u"ref-data"]
1155         if data_r:
1156             data_r_mean = mean(data_r)
1157             item.append(round(data_r_mean / 1000000, 2))
1158             data_r_stdev = stdev(data_r)
1159             item.append(round(data_r_stdev / 1000000, 2))
1160         else:
1161             data_r_mean = None
1162             data_r_stdev = None
1163             item.extend([None, None])
1164         data_c = tbl_dict[tst_name][u"cmp-data"]
1165         if data_c:
1166             data_c_mean = mean(data_c)
1167             item.append(round(data_c_mean / 1000000, 2))
1168             data_c_stdev = stdev(data_c)
1169             item.append(round(data_c_stdev / 1000000, 2))
1170         else:
1171             data_c_mean = None
1172             data_c_stdev = None
1173             item.extend([None, None])
1174         if data_r_mean and data_c_mean:
1175             delta, d_stdev = relative_change_stdev(
1176                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
1177             item.append(round(delta, 2))
1178             item.append(round(d_stdev, 2))
1179             tbl_lst.append(item)
1180
1181     # Sort the table according to the relative change
1182     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1183
1184     # Generate csv tables:
1185     csv_file = f"{table[u'output-file']}.csv"
1186     with open(csv_file, u"wt") as file_handler:
1187         file_handler.write(header_str)
1188         for test in tbl_lst:
1189             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
1190
1191     convert_csv_to_pretty_txt(csv_file, f"{table[u'output-file']}.txt")
1192
1193     # Generate html table:
1194     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
1195
1196
1197 def table_perf_trending_dash(table, input_data):
1198     """Generate the table(s) with algorithm:
1199     table_perf_trending_dash
1200     specified in the specification file.
1201
1202     :param table: Table to generate.
1203     :param input_data: Data to process.
1204     :type table: pandas.Series
1205     :type input_data: InputData
1206     """
1207
1208     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1209
1210     # Transform the data
1211     logging.info(
1212         f"    Creating the data set for the {table.get(u'type', u'')} "
1213         f"{table.get(u'title', u'')}."
1214     )
1215     data = input_data.filter_data(table, continue_on_error=True)
1216
1217     # Prepare the header of the tables
1218     header = [
1219         u"Test Case",
1220         u"Trend [Mpps]",
1221         u"Short-Term Change [%]",
1222         u"Long-Term Change [%]",
1223         u"Regressions [#]",
1224         u"Progressions [#]"
1225     ]
1226     header_str = u",".join(header) + u"\n"
1227
1228     # Prepare data to the table:
1229     tbl_dict = dict()
1230     for job, builds in table[u"data"].items():
1231         for build in builds:
1232             for tst_name, tst_data in data[job][str(build)].items():
1233                 if tst_name.lower() in table.get(u"ignore-list", list()):
1234                     continue
1235                 if tbl_dict.get(tst_name, None) is None:
1236                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1237                     if not groups:
1238                         continue
1239                     nic = groups.group(0)
1240                     tbl_dict[tst_name] = {
1241                         u"name": f"{nic}-{tst_data[u'name']}",
1242                         u"data": OrderedDict()
1243                     }
1244                 try:
1245                     tbl_dict[tst_name][u"data"][str(build)] = \
1246                         tst_data[u"result"][u"receive-rate"]
1247                 except (TypeError, KeyError):
1248                     pass  # No data in output.xml for this test
1249
1250     tbl_lst = list()
1251     for tst_name in tbl_dict:
1252         data_t = tbl_dict[tst_name][u"data"]
1253         if len(data_t) < 2:
1254             continue
1255
1256         classification_lst, avgs = classify_anomalies(data_t)
1257
1258         win_size = min(len(data_t), table[u"window"])
1259         long_win_size = min(len(data_t), table[u"long-trend-window"])
1260
1261         try:
1262             max_long_avg = max(
1263                 [x for x in avgs[-long_win_size:-win_size]
1264                  if not isnan(x)])
1265         except ValueError:
1266             max_long_avg = nan
1267         last_avg = avgs[-1]
1268         avg_week_ago = avgs[max(-win_size, -len(avgs))]
1269
1270         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
1271             rel_change_last = nan
1272         else:
1273             rel_change_last = round(
1274                 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
1275
1276         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
1277             rel_change_long = nan
1278         else:
1279             rel_change_long = round(
1280                 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
1281
1282         if classification_lst:
1283             if isnan(rel_change_last) and isnan(rel_change_long):
1284                 continue
1285             if isnan(last_avg) or isnan(rel_change_last) or \
1286                     isnan(rel_change_long):
1287                 continue
1288             tbl_lst.append(
1289                 [tbl_dict[tst_name][u"name"],
1290                  round(last_avg / 1000000, 2),
1291                  rel_change_last,
1292                  rel_change_long,
1293                  classification_lst[-win_size:].count(u"regression"),
1294                  classification_lst[-win_size:].count(u"progression")])
1295
1296     tbl_lst.sort(key=lambda rel: rel[0])
1297
1298     tbl_sorted = list()
1299     for nrr in range(table[u"window"], -1, -1):
1300         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
1301         for nrp in range(table[u"window"], -1, -1):
1302             tbl_out = [item for item in tbl_reg if item[5] == nrp]
1303             tbl_out.sort(key=lambda rel: rel[2])
1304             tbl_sorted.extend(tbl_out)
1305
1306     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1307
1308     logging.info(f"    Writing file: {file_name}")
1309     with open(file_name, u"wt") as file_handler:
1310         file_handler.write(header_str)
1311         for test in tbl_sorted:
1312             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1313
1314     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1315     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1316
1317
1318 def _generate_url(testbed, test_name):
1319     """Generate URL to a trending plot from the name of the test case.
1320
1321     :param testbed: The testbed used for testing.
1322     :param test_name: The name of the test case.
1323     :type testbed: str
1324     :type test_name: str
1325     :returns: The URL to the plot with the trending data for the given test
1326         case.
1327     :rtype str
1328     """
1329
1330     if u"x520" in test_name:
1331         nic = u"x520"
1332     elif u"x710" in test_name:
1333         nic = u"x710"
1334     elif u"xl710" in test_name:
1335         nic = u"xl710"
1336     elif u"xxv710" in test_name:
1337         nic = u"xxv710"
1338     elif u"vic1227" in test_name:
1339         nic = u"vic1227"
1340     elif u"vic1385" in test_name:
1341         nic = u"vic1385"
1342     elif u"x553" in test_name:
1343         nic = u"x553"
1344     else:
1345         nic = u""
1346
1347     if u"64b" in test_name:
1348         frame_size = u"64b"
1349     elif u"78b" in test_name:
1350         frame_size = u"78b"
1351     elif u"imix" in test_name:
1352         frame_size = u"imix"
1353     elif u"9000b" in test_name:
1354         frame_size = u"9000b"
1355     elif u"1518b" in test_name:
1356         frame_size = u"1518b"
1357     elif u"114b" in test_name:
1358         frame_size = u"114b"
1359     else:
1360         frame_size = u""
1361
1362     if u"1t1c" in test_name or \
1363         (u"-1c-" in test_name and
1364          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1365         cores = u"1t1c"
1366     elif u"2t2c" in test_name or \
1367          (u"-2c-" in test_name and
1368           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1369         cores = u"2t2c"
1370     elif u"4t4c" in test_name or \
1371          (u"-4c-" in test_name and
1372           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1373         cores = u"4t4c"
1374     elif u"2t1c" in test_name or \
1375          (u"-1c-" in test_name and
1376           testbed in (u"2n-skx", u"3n-skx")):
1377         cores = u"2t1c"
1378     elif u"4t2c" in test_name:
1379         cores = u"4t2c"
1380     elif u"8t4c" in test_name:
1381         cores = u"8t4c"
1382     else:
1383         cores = u""
1384
1385     if u"testpmd" in test_name:
1386         driver = u"testpmd"
1387     elif u"l3fwd" in test_name:
1388         driver = u"l3fwd"
1389     elif u"avf" in test_name:
1390         driver = u"avf"
1391     elif u"dnv" in testbed or u"tsh" in testbed:
1392         driver = u"ixgbe"
1393     else:
1394         driver = u"i40e"
1395
1396     if u"acl" in test_name or \
1397             u"macip" in test_name or \
1398             u"nat" in test_name or \
1399             u"policer" in test_name or \
1400             u"cop" in test_name:
1401         bsf = u"features"
1402     elif u"scale" in test_name:
1403         bsf = u"scale"
1404     elif u"base" in test_name:
1405         bsf = u"base"
1406     else:
1407         bsf = u"base"
1408
1409     if u"114b" in test_name and u"vhost" in test_name:
1410         domain = u"vts"
1411     elif u"testpmd" in test_name or u"l3fwd" in test_name:
1412         domain = u"dpdk"
1413     elif u"memif" in test_name:
1414         domain = u"container_memif"
1415     elif u"srv6" in test_name:
1416         domain = u"srv6"
1417     elif u"vhost" in test_name:
1418         domain = u"vhost"
1419         if u"vppl2xc" in test_name:
1420             driver += u"-vpp"
1421         else:
1422             driver += u"-testpmd"
1423         if u"lbvpplacp" in test_name:
1424             bsf += u"-link-bonding"
1425     elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1426         domain = u"nf_service_density_vnfc"
1427     elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1428         domain = u"nf_service_density_cnfc"
1429     elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1430         domain = u"nf_service_density_cnfp"
1431     elif u"ipsec" in test_name:
1432         domain = u"ipsec"
1433         if u"sw" in test_name:
1434             bsf += u"-sw"
1435         elif u"hw" in test_name:
1436             bsf += u"-hw"
1437     elif u"ethip4vxlan" in test_name:
1438         domain = u"ip4_tunnels"
1439     elif u"ip4base" in test_name or u"ip4scale" in test_name:
1440         domain = u"ip4"
1441     elif u"ip6base" in test_name or u"ip6scale" in test_name:
1442         domain = u"ip6"
1443     elif u"l2xcbase" in test_name or \
1444             u"l2xcscale" in test_name or \
1445             u"l2bdbasemaclrn" in test_name or \
1446             u"l2bdscale" in test_name or \
1447             u"l2patch" in test_name:
1448         domain = u"l2"
1449     else:
1450         domain = u""
1451
1452     file_name = u"-".join((domain, testbed, nic)) + u".html#"
1453     anchor_name = u"-".join((frame_size, cores, bsf, driver))
1454
1455     return file_name + anchor_name
1456
1457
1458 def table_perf_trending_dash_html(table, input_data):
1459     """Generate the table(s) with algorithm:
1460     table_perf_trending_dash_html specified in the specification
1461     file.
1462
1463     :param table: Table to generate.
1464     :param input_data: Data to process.
1465     :type table: dict
1466     :type input_data: InputData
1467     """
1468
1469     _ = input_data
1470
1471     if not table.get(u"testbed", None):
1472         logging.error(
1473             f"The testbed is not defined for the table "
1474             f"{table.get(u'title', u'')}."
1475         )
1476         return
1477
1478     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1479
1480     try:
1481         with open(table[u"input-file"], u'rt') as csv_file:
1482             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1483     except KeyError:
1484         logging.warning(u"The input file is not defined.")
1485         return
1486     except csv.Error as err:
1487         logging.warning(
1488             f"Not possible to process the file {table[u'input-file']}.\n"
1489             f"{repr(err)}"
1490         )
1491         return
1492
1493     # Table:
1494     dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1495
1496     # Table header:
1497     trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1498     for idx, item in enumerate(csv_lst[0]):
1499         alignment = u"left" if idx == 0 else u"center"
1500         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1501         thead.text = item
1502
1503     # Rows:
1504     colors = {
1505         u"regression": (
1506             u"#ffcccc",
1507             u"#ff9999"
1508         ),
1509         u"progression": (
1510             u"#c6ecc6",
1511             u"#9fdf9f"
1512         ),
1513         u"normal": (
1514             u"#e9f1fb",
1515             u"#d4e4f7"
1516         )
1517     }
1518     for r_idx, row in enumerate(csv_lst[1:]):
1519         if int(row[4]):
1520             color = u"regression"
1521         elif int(row[5]):
1522             color = u"progression"
1523         else:
1524             color = u"normal"
1525         trow = ET.SubElement(
1526             dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
1527         )
1528
1529         # Columns:
1530         for c_idx, item in enumerate(row):
1531             tdata = ET.SubElement(
1532                 trow,
1533                 u"td",
1534                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1535             )
1536             # Name:
1537             if c_idx == 0:
1538                 ref = ET.SubElement(
1539                     tdata,
1540                     u"a",
1541                     attrib=dict(
1542                         href=f"../trending/"
1543                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1544                     )
1545                 )
1546                 ref.text = item
1547             else:
1548                 tdata.text = item
1549     try:
1550         with open(table[u"output-file"], u'w') as html_file:
1551             logging.info(f"    Writing file: {table[u'output-file']}")
1552             html_file.write(u".. raw:: html\n\n\t")
1553             html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
1554             html_file.write(u"\n\t<p><br><br></p>\n")
1555     except KeyError:
1556         logging.warning(u"The output file is not defined.")
1557         return
1558
1559
1560 def table_last_failed_tests(table, input_data):
1561     """Generate the table(s) with algorithm: table_last_failed_tests
1562     specified in the specification file.
1563
1564     :param table: Table to generate.
1565     :param input_data: Data to process.
1566     :type table: pandas.Series
1567     :type input_data: InputData
1568     """
1569
1570     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1571
1572     # Transform the data
1573     logging.info(
1574         f"    Creating the data set for the {table.get(u'type', u'')} "
1575         f"{table.get(u'title', u'')}."
1576     )
1577
1578     data = input_data.filter_data(table, continue_on_error=True)
1579
1580     if data is None or data.empty:
1581         logging.warning(
1582             f"    No data for the {table.get(u'type', u'')} "
1583             f"{table.get(u'title', u'')}."
1584         )
1585         return
1586
1587     tbl_list = list()
1588     for job, builds in table[u"data"].items():
1589         for build in builds:
1590             build = str(build)
1591             try:
1592                 version = input_data.metadata(job, build).get(u"version", u"")
1593             except KeyError:
1594                 logging.error(f"Data for {job}: {build} is not present.")
1595                 return
1596             tbl_list.append(build)
1597             tbl_list.append(version)
1598             failed_tests = list()
1599             passed = 0
1600             failed = 0
1601             for tst_data in data[job][build].values:
1602                 if tst_data[u"status"] != u"FAIL":
1603                     passed += 1
1604                     continue
1605                 failed += 1
1606                 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1607                 if not groups:
1608                     continue
1609                 nic = groups.group(0)
1610                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
1611             tbl_list.append(str(passed))
1612             tbl_list.append(str(failed))
1613             tbl_list.extend(failed_tests)
1614
1615     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1616     logging.info(f"    Writing file: {file_name}")
1617     with open(file_name, u"wt") as file_handler:
1618         for test in tbl_list:
1619             file_handler.write(test + u'\n')
1620
1621
1622 def table_failed_tests(table, input_data):
1623     """Generate the table(s) with algorithm: table_failed_tests
1624     specified in the specification file.
1625
1626     :param table: Table to generate.
1627     :param input_data: Data to process.
1628     :type table: pandas.Series
1629     :type input_data: InputData
1630     """
1631
1632     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1633
1634     # Transform the data
1635     logging.info(
1636         f"    Creating the data set for the {table.get(u'type', u'')} "
1637         f"{table.get(u'title', u'')}."
1638     )
1639     data = input_data.filter_data(table, continue_on_error=True)
1640
1641     # Prepare the header of the tables
1642     header = [
1643         u"Test Case",
1644         u"Failures [#]",
1645         u"Last Failure [Time]",
1646         u"Last Failure [VPP-Build-Id]",
1647         u"Last Failure [CSIT-Job-Build-Id]"
1648     ]
1649
1650     # Generate the data for the table according to the model in the table
1651     # specification
1652
1653     now = dt.utcnow()
1654     timeperiod = timedelta(int(table.get(u"window", 7)))
1655
1656     tbl_dict = dict()
1657     for job, builds in table[u"data"].items():
1658         for build in builds:
1659             build = str(build)
1660             for tst_name, tst_data in data[job][build].items():
1661                 if tst_name.lower() in table.get(u"ignore-list", list()):
1662                     continue
1663                 if tbl_dict.get(tst_name, None) is None:
1664                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1665                     if not groups:
1666                         continue
1667                     nic = groups.group(0)
1668                     tbl_dict[tst_name] = {
1669                         u"name": f"{nic}-{tst_data[u'name']}",
1670                         u"data": OrderedDict()
1671                     }
1672                 try:
1673                     generated = input_data.metadata(job, build).\
1674                         get(u"generated", u"")
1675                     if not generated:
1676                         continue
1677                     then = dt.strptime(generated, u"%Y%m%d %H:%M")
1678                     if (now - then) <= timeperiod:
1679                         tbl_dict[tst_name][u"data"][build] = (
1680                             tst_data[u"status"],
1681                             generated,
1682                             input_data.metadata(job, build).get(u"version",
1683                                                                 u""),
1684                             build
1685                         )
1686                 except (TypeError, KeyError) as err:
1687                     logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
1688
1689     max_fails = 0
1690     tbl_lst = list()
1691     for tst_data in tbl_dict.values():
1692         fails_nr = 0
1693         fails_last_date = u""
1694         fails_last_vpp = u""
1695         fails_last_csit = u""
1696         for val in tst_data[u"data"].values():
1697             if val[0] == u"FAIL":
1698                 fails_nr += 1
1699                 fails_last_date = val[1]
1700                 fails_last_vpp = val[2]
1701                 fails_last_csit = val[3]
1702         if fails_nr:
1703             max_fails = fails_nr if fails_nr > max_fails else max_fails
1704             tbl_lst.append(
1705                 [
1706                     tst_data[u"name"],
1707                     fails_nr,
1708                     fails_last_date,
1709                     fails_last_vpp,
1710                     f"mrr-daily-build-{fails_last_csit}"
1711                 ]
1712             )
1713
1714     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1715     tbl_sorted = list()
1716     for nrf in range(max_fails, -1, -1):
1717         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1718         tbl_sorted.extend(tbl_fails)
1719
1720     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1721     logging.info(f"    Writing file: {file_name}")
1722     with open(file_name, u"wt") as file_handler:
1723         file_handler.write(u",".join(header) + u"\n")
1724         for test in tbl_sorted:
1725             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1726
1727     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1728     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1729
1730
1731 def table_failed_tests_html(table, input_data):
1732     """Generate the table(s) with algorithm: table_failed_tests_html
1733     specified in the specification file.
1734
1735     :param table: Table to generate.
1736     :param input_data: Data to process.
1737     :type table: pandas.Series
1738     :type input_data: InputData
1739     """
1740
1741     _ = input_data
1742
1743     if not table.get(u"testbed", None):
1744         logging.error(
1745             f"The testbed is not defined for the table "
1746             f"{table.get(u'title', u'')}."
1747         )
1748         return
1749
1750     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1751
1752     try:
1753         with open(table[u"input-file"], u'rt') as csv_file:
1754             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1755     except KeyError:
1756         logging.warning(u"The input file is not defined.")
1757         return
1758     except csv.Error as err:
1759         logging.warning(
1760             f"Not possible to process the file {table[u'input-file']}.\n"
1761             f"{repr(err)}"
1762         )
1763         return
1764
1765     # Table:
1766     failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1767
1768     # Table header:
1769     trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1770     for idx, item in enumerate(csv_lst[0]):
1771         alignment = u"left" if idx == 0 else u"center"
1772         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1773         thead.text = item
1774
1775     # Rows:
1776     colors = (u"#e9f1fb", u"#d4e4f7")
1777     for r_idx, row in enumerate(csv_lst[1:]):
1778         background = colors[r_idx % 2]
1779         trow = ET.SubElement(
1780             failed_tests, u"tr", attrib=dict(bgcolor=background)
1781         )
1782
1783         # Columns:
1784         for c_idx, item in enumerate(row):
1785             tdata = ET.SubElement(
1786                 trow,
1787                 u"td",
1788                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1789             )
1790             # Name:
1791             if c_idx == 0:
1792                 ref = ET.SubElement(
1793                     tdata,
1794                     u"a",
1795                     attrib=dict(
1796                         href=f"../trending/"
1797                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1798                     )
1799                 )
1800                 ref.text = item
1801             else:
1802                 tdata.text = item
1803     try:
1804         with open(table[u"output-file"], u'w') as html_file:
1805             logging.info(f"    Writing file: {table[u'output-file']}")
1806             html_file.write(u".. raw:: html\n\n\t")
1807             html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
1808             html_file.write(u"\n\t<p><br><br></p>\n")
1809     except KeyError:
1810         logging.warning(u"The output file is not defined.")
1811         return