resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from collections import OrderedDict
  23 from xml.etree import ElementTree as ET
  24 from datetime import datetime as dt
  25 from datetime import timedelta
  26
  27 import plotly.graph_objects as go
  28 import plotly.offline as ploff
  29 import pandas as pd
  30
  31 from numpy import nan, isnan
  32
  33 from pal_utils import mean, stdev, classify_anomalies, \
  34     convert_csv_to_pretty_txt, relative_change_stdev
  35
  36
  37 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
  38
  39
  40 def generate_tables(spec, data):
  41     """Generate all tables specified in the specification file.
  42
  43     :param spec: Specification read from the specification file.
  44     :param data: Data to process.
  45     :type spec: Specification
  46     :type data: InputData
  47     """
  48
  49     generator = {
  50         u"table_merged_details": table_merged_details,
  51         u"table_perf_comparison": table_perf_comparison,
  52         u"table_perf_comparison_nic": table_perf_comparison_nic,
  53         u"table_nics_comparison": table_nics_comparison,
  54         u"table_soak_vs_ndr": table_soak_vs_ndr,
  55         u"table_perf_trending_dash": table_perf_trending_dash,
  56         u"table_perf_trending_dash_html": table_perf_trending_dash_html,
  57         u"table_last_failed_tests": table_last_failed_tests,
  58         u"table_failed_tests": table_failed_tests,
  59         u"table_failed_tests_html": table_failed_tests_html,
  60         u"table_oper_data_html": table_oper_data_html
  61     }
  62
  63     logging.info(u"Generating the tables ...")
  64     for table in spec.tables:
  65         try:
  66             generator[table[u"algorithm"]](table, data)
  67         except NameError as err:
  68             logging.error(
  69                 f"Probably algorithm {table[u'algorithm']} is not defined: "
  70                 f"{repr(err)}"
  71             )
  72     logging.info(u"Done.")
  73
  74
  75 def table_oper_data_html(table, input_data):
  76     """Generate the table(s) with algorithm: html_table_oper_data
  77     specified in the specification file.
  78
  79     :param table: Table to generate.
  80     :param input_data: Data to process.
  81     :type table: pandas.Series
  82     :type input_data: InputData
  83     """
  84
  85     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
  86     # Transform the data
  87     logging.info(
  88         f"    Creating the data set for the {table.get(u'type', u'')} "
  89         f"{table.get(u'title', u'')}."
  90     )
  91     data = input_data.filter_data(
  92         table,
  93         params=[u"name", u"parent", u"show-run", u"type"],
  94         continue_on_error=True
  95     )
  96     if data.empty:
  97         return
  98     data = input_data.merge_data(data)
  99
 100     sort_tests = table.get(u"sort", None)
 101     if sort_tests:
 102         args = dict(
 103             inplace=True,
 104             ascending=(sort_tests == u"ascending")
 105         )
 106         data.sort_index(**args)
 107
 108     suites = input_data.filter_data(
 109         table,
 110         continue_on_error=True,
 111         data_set=u"suites"
 112     )
 113     if suites.empty:
 114         return
 115     suites = input_data.merge_data(suites)
 116
 117     def _generate_html_table(tst_data):
 118         """Generate an HTML table with operational data for the given test.
 119
 120         :param tst_data: Test data to be used to generate the table.
 121         :type tst_data: pandas.Series
 122         :returns: HTML table with operational data.
 123         :rtype: str
 124         """
 125
 126         colors = {
 127             u"header": u"#7eade7",
 128             u"empty": u"#ffffff",
 129             u"body": (u"#e9f1fb", u"#d4e4f7")
 130         }
 131
 132         tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
 133
 134         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
 135         thead = ET.SubElement(
 136             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 137         )
 138         thead.text = tst_data[u"name"]
 139
 140         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 141         thead = ET.SubElement(
 142             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 143         )
 144         thead.text = u"\t"
 145
 146         if tst_data.get(u"show-run", u"No Data") == u"No Data":
 147             trow = ET.SubElement(
 148                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 149             )
 150             tcol = ET.SubElement(
 151                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 152             )
 153             tcol.text = u"No Data"
 154
 155             trow = ET.SubElement(
 156                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 157             )
 158             thead = ET.SubElement(
 159                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 160             )
 161             font = ET.SubElement(
 162                 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 163             )
 164             font.text = u"."
 165             return str(ET.tostring(tbl, encoding=u"unicode"))
 166
 167         tbl_hdr = (
 168             u"Name",
 169             u"Nr of Vectors",
 170             u"Nr of Packets",
 171             u"Suspends",
 172             u"Cycles per Packet",
 173             u"Average Vector Size"
 174         )
 175
 176         for dut_data in tst_data[u"show-run"].values():
 177             trow = ET.SubElement(
 178                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 179             )
 180             tcol = ET.SubElement(
 181                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 182             )
 183             if dut_data.get(u"threads", None) is None:
 184                 tcol.text = u"No Data"
 185                 continue
 186
 187             bold = ET.SubElement(tcol, u"b")
 188             bold.text = (
 189                 f"Host IP: {dut_data.get(u'host', '')}, "
 190                 f"Socket: {dut_data.get(u'socket', '')}"
 191             )
 192             trow = ET.SubElement(
 193                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 194             )
 195             thead = ET.SubElement(
 196                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 197             )
 198             thead.text = u"\t"
 199
 200             for thread_nr, thread in dut_data[u"threads"].items():
 201                 trow = ET.SubElement(
 202                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 203                 )
 204                 tcol = ET.SubElement(
 205                     trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 206                 )
 207                 bold = ET.SubElement(tcol, u"b")
 208                 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
 209                 trow = ET.SubElement(
 210                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 211                 )
 212                 for idx, col in enumerate(tbl_hdr):
 213                     tcol = ET.SubElement(
 214                         trow, u"td",
 215                         attrib=dict(align=u"right" if idx else u"left")
 216                     )
 217                     font = ET.SubElement(
 218                         tcol, u"font", attrib=dict(size=u"2")
 219                     )
 220                     bold = ET.SubElement(font, u"b")
 221                     bold.text = col
 222                 for row_nr, row in enumerate(thread):
 223                     trow = ET.SubElement(
 224                         tbl, u"tr",
 225                         attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
 226                     )
 227                     for idx, col in enumerate(row):
 228                         tcol = ET.SubElement(
 229                             trow, u"td",
 230                             attrib=dict(align=u"right" if idx else u"left")
 231                         )
 232                         font = ET.SubElement(
 233                             tcol, u"font", attrib=dict(size=u"2")
 234                         )
 235                         if isinstance(col, float):
 236                             font.text = f"{col:.2f}"
 237                         else:
 238                             font.text = str(col)
 239                 trow = ET.SubElement(
 240                     tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 241                 )
 242                 thead = ET.SubElement(
 243                     trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 244                 )
 245                 thead.text = u"\t"
 246
 247         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 248         thead = ET.SubElement(
 249             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 250         )
 251         font = ET.SubElement(
 252             thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 253         )
 254         font.text = u"."
 255
 256         return str(ET.tostring(tbl, encoding=u"unicode"))
 257
 258     for suite in suites.values:
 259         html_table = str()
 260         for test_data in data.values:
 261             if test_data[u"parent"] not in suite[u"name"]:
 262                 continue
 263             html_table += _generate_html_table(test_data)
 264         if not html_table:
 265             continue
 266         try:
 267             file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
 268             with open(f"{file_name}", u'w') as html_file:
 269                 logging.info(f"    Writing file: {file_name}")
 270                 html_file.write(u".. raw:: html\n\n\t")
 271                 html_file.write(html_table)
 272                 html_file.write(u"\n\t<p><br><br></p>\n")
 273         except KeyError:
 274             logging.warning(u"The output file is not defined.")
 275             return
 276     logging.info(u"  Done.")
 277
 278
 279 def table_merged_details(table, input_data):
 280     """Generate the table(s) with algorithm: table_merged_details
 281     specified in the specification file.
 282
 283     :param table: Table to generate.
 284     :param input_data: Data to process.
 285     :type table: pandas.Series
 286     :type input_data: InputData
 287     """
 288
 289     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 290
 291     # Transform the data
 292     logging.info(
 293         f"    Creating the data set for the {table.get(u'type', u'')} "
 294         f"{table.get(u'title', u'')}."
 295     )
 296     data = input_data.filter_data(table, continue_on_error=True)
 297     data = input_data.merge_data(data)
 298
 299     sort_tests = table.get(u"sort", None)
 300     if sort_tests:
 301         args = dict(
 302             inplace=True,
 303             ascending=(sort_tests == u"ascending")
 304         )
 305         data.sort_index(**args)
 306
 307     suites = input_data.filter_data(
 308         table, continue_on_error=True, data_set=u"suites")
 309     suites = input_data.merge_data(suites)
 310
 311     # Prepare the header of the tables
 312     header = list()
 313     for column in table[u"columns"]:
 314         header.append(
 315             u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
 316         )
 317
 318     for suite in suites.values:
 319         # Generate data
 320         suite_name = suite[u"name"]
 321         table_lst = list()
 322         for test in data.keys():
 323             if data[test][u"parent"] not in suite_name:
 324                 continue
 325             row_lst = list()
 326             for column in table[u"columns"]:
 327                 try:
 328                     col_data = str(data[test][column[
 329                         u"data"].split(u" ")[1]]).replace(u'"', u'""')
 330                     # Do not include tests with "Test Failed" in test message
 331                     if u"Test Failed" in col_data:
 332                         continue
 333                     col_data = col_data.replace(
 334                         u"No Data", u"Not Captured     "
 335                     )
 336                     if column[u"data"].split(u" ")[1] in (u"name", ):
 337                         if len(col_data) > 30:
 338                             col_data_lst = col_data.split(u"-")
 339                             half = int(len(col_data_lst) / 2)
 340                             col_data = f"{u'-'.join(col_data_lst[:half])}" \
 341                                        f"- |br| " \
 342                                        f"{u'-'.join(col_data_lst[half:])}"
 343                         col_data = f" |prein| {col_data} |preout| "
 344                     elif column[u"data"].split(u" ")[1] in (u"msg", ):
 345                         # Temporary solution: remove NDR results from message:
 346                         if bool(table.get(u'remove-ndr', False)):
 347                             try:
 348                                 col_data = col_data.split(u" |br| ", 1)[1]
 349                             except IndexError:
 350                                 pass
 351                         col_data = f" |prein| {col_data} |preout| "
 352                     elif column[u"data"].split(u" ")[1] in \
 353                             (u"conf-history", u"show-run"):
 354                         col_data = col_data.replace(u" |br| ", u"", 1)
 355                         col_data = f" |prein| {col_data[:-5]} |preout| "
 356                     row_lst.append(f'"{col_data}"')
 357                 except KeyError:
 358                     row_lst.append(u'"Not captured"')
 359             if len(row_lst) == len(table[u"columns"]):
 360                 table_lst.append(row_lst)
 361
 362         # Write the data to file
 363         if table_lst:
 364             separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
 365             file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
 366             logging.info(f"      Writing file: {file_name}")
 367             with open(file_name, u"wt") as file_handler:
 368                 file_handler.write(u",".join(header) + u"\n")
 369                 for item in table_lst:
 370                     file_handler.write(u",".join(item) + u"\n")
 371
 372     logging.info(u"  Done.")
 373
 374
 375 def _tpc_modify_test_name(test_name):
 376     """Modify a test name by replacing its parts.
 377
 378     :param test_name: Test name to be modified.
 379     :type test_name: str
 380     :returns: Modified test name.
 381     :rtype: str
 382     """
 383     test_name_mod = test_name.\
 384         replace(u"-ndrpdrdisc", u""). \
 385         replace(u"-ndrpdr", u"").\
 386         replace(u"-pdrdisc", u""). \
 387         replace(u"-ndrdisc", u"").\
 388         replace(u"-pdr", u""). \
 389         replace(u"-ndr", u""). \
 390         replace(u"1t1c", u"1c").\
 391         replace(u"2t1c", u"1c"). \
 392         replace(u"2t2c", u"2c").\
 393         replace(u"4t2c", u"2c"). \
 394         replace(u"4t4c", u"4c").\
 395         replace(u"8t4c", u"4c")
 396
 397     return re.sub(REGEX_NIC, u"", test_name_mod)
 398
 399
 400 def _tpc_modify_displayed_test_name(test_name):
 401     """Modify a test name which is displayed in a table by replacing its parts.
 402
 403     :param test_name: Test name to be modified.
 404     :type test_name: str
 405     :returns: Modified test name.
 406     :rtype: str
 407     """
 408     return test_name.\
 409         replace(u"1t1c", u"1c").\
 410         replace(u"2t1c", u"1c"). \
 411         replace(u"2t2c", u"2c").\
 412         replace(u"4t2c", u"2c"). \
 413         replace(u"4t4c", u"4c").\
 414         replace(u"8t4c", u"4c")
 415
 416
 417 def _tpc_insert_data(target, src, include_tests):
 418     """Insert src data to the target structure.
 419
 420     :param target: Target structure where the data is placed.
 421     :param src: Source data to be placed into the target stucture.
 422     :param include_tests: Which results will be included (MRR, NDR, PDR).
 423     :type target: list
 424     :type src: dict
 425     :type include_tests: str
 426     """
 427     try:
 428         if include_tests == u"MRR":
 429             target.append(src[u"result"][u"receive-rate"])
 430         elif include_tests == u"PDR":
 431             target.append(src[u"throughput"][u"PDR"][u"LOWER"])
 432         elif include_tests == u"NDR":
 433             target.append(src[u"throughput"][u"NDR"][u"LOWER"])
 434     except (KeyError, TypeError):
 435         pass
 436
 437
 438 def _tpc_sort_table(table):
 439     """Sort the table this way:
 440
 441     1. Put "New in CSIT-XXXX" at the first place.
 442     2. Put "See footnote" at the second place.
 443     3. Sort the rest by "Delta".
 444
 445     :param table: Table to sort.
 446     :type table: list
 447     :returns: Sorted table.
 448     :rtype: list
 449     """
 450
 451     tbl_new = list()
 452     tbl_see = list()
 453     tbl_delta = list()
 454     for item in table:
 455         if isinstance(item[-1], str):
 456             if u"New in CSIT" in item[-1]:
 457                 tbl_new.append(item)
 458             elif u"See footnote" in item[-1]:
 459                 tbl_see.append(item)
 460         else:
 461             tbl_delta.append(item)
 462
 463     # Sort the tables:
 464     tbl_new.sort(key=lambda rel: rel[0], reverse=False)
 465     tbl_see.sort(key=lambda rel: rel[0], reverse=False)
 466     tbl_see.sort(key=lambda rel: rel[-2], reverse=False)
 467     tbl_delta.sort(key=lambda rel: rel[0], reverse=False)
 468     tbl_delta.sort(key=lambda rel: rel[-2], reverse=True)
 469
 470     # Put the tables together:
 471     table = list()
 472     # We do not want "New in CSIT":
 473     # table.extend(tbl_new)
 474     table.extend(tbl_see)
 475     table.extend(tbl_delta)
 476
 477     return table
 478
 479
 480 def _tpc_generate_html_table(header, data, output_file_name):
 481     """Generate html table from input data with simple sorting possibility.
 482
 483     :param header: Table header.
 484     :param data: Input data to be included in the table. It is a list of lists.
 485         Inner lists are rows in the table. All inner lists must be of the same
 486         length. The length of these lists must be the same as the length of the
 487         header.
 488     :param output_file_name: The name (relative or full path) where the
 489         generated html table is written.
 490     :type header: list
 491     :type data: list of lists
 492     :type output_file_name: str
 493     """
 494
 495     df_data = pd.DataFrame(data, columns=header)
 496
 497     df_sorted = [df_data.sort_values(
 498         by=[key, header[0]], ascending=[True, True]
 499         if key != header[0] else [False, True]) for key in header]
 500     df_sorted_rev = [df_data.sort_values(
 501         by=[key, header[0]], ascending=[False, True]
 502         if key != header[0] else [True, True]) for key in header]
 503     df_sorted.extend(df_sorted_rev)
 504
 505     fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
 506                    for idx in range(len(df_data))]]
 507     table_header = dict(
 508         values=[f"<b>{item}</b>" for item in header],
 509         fill_color=u"#7eade7",
 510         align=[u"left", u"center"]
 511     )
 512
 513     fig = go.Figure()
 514
 515     for table in df_sorted:
 516         columns = [table.get(col) for col in header]
 517         fig.add_trace(
 518             go.Table(
 519                 columnwidth=[30, 10],
 520                 header=table_header,
 521                 cells=dict(
 522                     values=columns,
 523                     fill_color=fill_color,
 524                     align=[u"left", u"right"]
 525                 )
 526             )
 527         )
 528
 529     buttons = list()
 530     menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
 531     menu_items_rev = [f"<b>{itm}</b> (descending)" for itm in header]
 532     menu_items.extend(menu_items_rev)
 533     for idx, hdr in enumerate(menu_items):
 534         visible = [False, ] * len(menu_items)
 535         visible[idx] = True
 536         buttons.append(
 537             dict(
 538                 label=hdr.replace(u" [Mpps]", u""),
 539                 method=u"update",
 540                 args=[{u"visible": visible}],
 541             )
 542         )
 543
 544     fig.update_layout(
 545         updatemenus=[
 546             go.layout.Updatemenu(
 547                 type=u"dropdown",
 548                 direction=u"down",
 549                 x=0.03,
 550                 xanchor=u"left",
 551                 y=1.045,
 552                 yanchor=u"top",
 553                 active=len(menu_items) - 2,
 554                 buttons=list(buttons)
 555             )
 556         ],
 557         annotations=[
 558             go.layout.Annotation(
 559                 text=u"<b>Sort by:</b>",
 560                 x=0,
 561                 xref=u"paper",
 562                 y=1.035,
 563                 yref=u"paper",
 564                 align=u"left",
 565                 showarrow=False
 566             )
 567         ]
 568     )
 569
 570     ploff.plot(fig, show_link=False, auto_open=False, filename=output_file_name)
 571
 572
 573 def table_perf_comparison(table, input_data):
 574     """Generate the table(s) with algorithm: table_perf_comparison
 575     specified in the specification file.
 576
 577     :param table: Table to generate.
 578     :param input_data: Data to process.
 579     :type table: pandas.Series
 580     :type input_data: InputData
 581     """
 582
 583     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 584
 585     # Transform the data
 586     logging.info(
 587         f"    Creating the data set for the {table.get(u'type', u'')} "
 588         f"{table.get(u'title', u'')}."
 589     )
 590     data = input_data.filter_data(table, continue_on_error=True)
 591
 592     # Prepare the header of the tables
 593     try:
 594         header = [u"Test case", ]
 595
 596         if table[u"include-tests"] == u"MRR":
 597             hdr_param = u"Rec Rate"
 598         else:
 599             hdr_param = u"Thput"
 600
 601         history = table.get(u"history", list())
 602         for item in history:
 603             header.extend(
 604                 [
 605                     f"{item[u'title']} {hdr_param} [Mpps]",
 606                     f"{item[u'title']} Stdev [Mpps]"
 607                 ]
 608             )
 609         header.extend(
 610             [
 611                 f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
 612                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
 613                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
 614                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
 615                 u"Delta [%]",
 616                 u"Stdev of delta [%]"
 617             ]
 618         )
 619         header_str = u",".join(header) + u"\n"
 620     except (AttributeError, KeyError) as err:
 621         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 622         return
 623
 624     # Prepare data to the table:
 625     tbl_dict = dict()
 626     # topo = ""
 627     for job, builds in table[u"reference"][u"data"].items():
 628         # topo = u"2n-skx" if u"2n-skx" in job else u""
 629         for build in builds:
 630             for tst_name, tst_data in data[job][str(build)].items():
 631                 tst_name_mod = _tpc_modify_test_name(tst_name)
 632                 if (u"across topologies" in table[u"title"].lower() or
 633                         (u" 3n-" in table[u"title"].lower() and
 634                          u" 2n-" in table[u"title"].lower())):
 635                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 636                 if tbl_dict.get(tst_name_mod, None) is None:
 637                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
 638                     nic = groups.group(0) if groups else u""
 639                     name = \
 640                         f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 641                     if u"across testbeds" in table[u"title"].lower() or \
 642                             u"across topologies" in table[u"title"].lower():
 643                         name = _tpc_modify_displayed_test_name(name)
 644                     tbl_dict[tst_name_mod] = {
 645                         u"name": name,
 646                         u"ref-data": list(),
 647                         u"cmp-data": list()
 648                     }
 649                 _tpc_insert_data(target=tbl_dict[tst_name_mod][u"ref-data"],
 650                                  src=tst_data,
 651                                  include_tests=table[u"include-tests"])
 652
 653     replacement = table[u"reference"].get(u"data-replacement", None)
 654     if replacement:
 655         create_new_list = True
 656         rpl_data = input_data.filter_data(
 657             table, data=replacement, continue_on_error=True)
 658         for job, builds in replacement.items():
 659             for build in builds:
 660                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 661                     tst_name_mod = _tpc_modify_test_name(tst_name)
 662                     if (u"across topologies" in table[u"title"].lower() or
 663                             (u" 3n-" in table[u"title"].lower() and
 664                              u" 2n-" in table[u"title"].lower())):
 665                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 666                     if tbl_dict.get(tst_name_mod, None) is None:
 667                         name = \
 668                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 669                         if u"across testbeds" in table[u"title"].lower() or \
 670                                 u"across topologies" in table[u"title"].lower():
 671                             name = _tpc_modify_displayed_test_name(name)
 672                         tbl_dict[tst_name_mod] = {
 673                             u"name": name,
 674                             u"ref-data": list(),
 675                             u"cmp-data": list()
 676                         }
 677                     if create_new_list:
 678                         create_new_list = False
 679                         tbl_dict[tst_name_mod][u"ref-data"] = list()
 680
 681                     _tpc_insert_data(
 682                         target=tbl_dict[tst_name_mod][u"ref-data"],
 683                         src=tst_data,
 684                         include_tests=table[u"include-tests"]
 685                     )
 686
 687     for job, builds in table[u"compare"][u"data"].items():
 688         for build in builds:
 689             for tst_name, tst_data in data[job][str(build)].items():
 690                 tst_name_mod = _tpc_modify_test_name(tst_name)
 691                 if (u"across topologies" in table[u"title"].lower() or
 692                         (u" 3n-" in table[u"title"].lower() and
 693                          u" 2n-" in table[u"title"].lower())):
 694                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 695                 if tbl_dict.get(tst_name_mod, None) is None:
 696                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
 697                     nic = groups.group(0) if groups else u""
 698                     name = \
 699                         f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 700                     if u"across testbeds" in table[u"title"].lower() or \
 701                             u"across topologies" in table[u"title"].lower():
 702                         name = _tpc_modify_displayed_test_name(name)
 703                     tbl_dict[tst_name_mod] = {
 704                         u"name": name,
 705                         u"ref-data": list(),
 706                         u"cmp-data": list()
 707                     }
 708                 _tpc_insert_data(
 709                     target=tbl_dict[tst_name_mod][u"cmp-data"],
 710                     src=tst_data,
 711                     include_tests=table[u"include-tests"]
 712                 )
 713
 714     replacement = table[u"compare"].get(u"data-replacement", None)
 715     if replacement:
 716         create_new_list = True
 717         rpl_data = input_data.filter_data(
 718             table, data=replacement, continue_on_error=True)
 719         for job, builds in replacement.items():
 720             for build in builds:
 721                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 722                     tst_name_mod = _tpc_modify_test_name(tst_name)
 723                     if (u"across topologies" in table[u"title"].lower() or
 724                             (u" 3n-" in table[u"title"].lower() and
 725                              u" 2n-" in table[u"title"].lower())):
 726                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 727                     if tbl_dict.get(tst_name_mod, None) is None:
 728                         name = \
 729                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 730                         if u"across testbeds" in table[u"title"].lower() or \
 731                                 u"across topologies" in table[u"title"].lower():
 732                             name = _tpc_modify_displayed_test_name(name)
 733                         tbl_dict[tst_name_mod] = {
 734                             u"name": name,
 735                             u"ref-data": list(),
 736                             u"cmp-data": list()
 737                         }
 738                     if create_new_list:
 739                         create_new_list = False
 740                         tbl_dict[tst_name_mod][u"cmp-data"] = list()
 741
 742                     _tpc_insert_data(
 743                         target=tbl_dict[tst_name_mod][u"cmp-data"],
 744                         src=tst_data,
 745                         include_tests=table[u"include-tests"]
 746                     )
 747
 748     for item in history:
 749         for job, builds in item[u"data"].items():
 750             for build in builds:
 751                 for tst_name, tst_data in data[job][str(build)].items():
 752                     tst_name_mod = _tpc_modify_test_name(tst_name)
 753                     if (u"across topologies" in table[u"title"].lower() or
 754                             (u" 3n-" in table[u"title"].lower() and
 755                              u" 2n-" in table[u"title"].lower())):
 756                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 757                     if tbl_dict.get(tst_name_mod, None) is None:
 758                         continue
 759                     if tbl_dict[tst_name_mod].get(u"history", None) is None:
 760                         tbl_dict[tst_name_mod][u"history"] = OrderedDict()
 761                     if tbl_dict[tst_name_mod][u"history"].\
 762                             get(item[u"title"], None) is None:
 763                         tbl_dict[tst_name_mod][u"history"][item[
 764                             u"title"]] = list()
 765                     try:
 766                         if table[u"include-tests"] == u"MRR":
 767                             res = tst_data[u"result"][u"receive-rate"]
 768                         elif table[u"include-tests"] == u"PDR":
 769                             res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
 770                         elif table[u"include-tests"] == u"NDR":
 771                             res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
 772                         else:
 773                             continue
 774                         tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
 775                             append(res)
 776                     except (TypeError, KeyError):
 777                         pass
 778
 779     tbl_lst = list()
 780     footnote = False
 781     for tst_name in tbl_dict:
 782         item = [tbl_dict[tst_name][u"name"], ]
 783         if history:
 784             if tbl_dict[tst_name].get(u"history", None) is not None:
 785                 for hist_data in tbl_dict[tst_name][u"history"].values():
 786                     if hist_data:
 787                         item.append(round(mean(hist_data) / 1000000, 2))
 788                         item.append(round(stdev(hist_data) / 1000000, 2))
 789                     else:
 790                         item.extend([u"Not tested", u"Not tested"])
 791             else:
 792                 item.extend([u"Not tested", u"Not tested"])
 793         data_r = tbl_dict[tst_name][u"ref-data"]
 794         if data_r:
 795             data_r_mean = mean(data_r)
 796             item.append(round(data_r_mean / 1000000, 2))
 797             data_r_stdev = stdev(data_r)
 798             item.append(round(data_r_stdev / 1000000, 2))
 799         else:
 800             data_r_mean = None
 801             data_r_stdev = None
 802             item.extend([u"Not tested", u"Not tested"])
 803         data_c = tbl_dict[tst_name][u"cmp-data"]
 804         if data_c:
 805             data_c_mean = mean(data_c)
 806             item.append(round(data_c_mean / 1000000, 2))
 807             data_c_stdev = stdev(data_c)
 808             item.append(round(data_c_stdev / 1000000, 2))
 809         else:
 810             data_c_mean = None
 811             data_c_stdev = None
 812             item.extend([u"Not tested", u"Not tested"])
 813         if item[-2] == u"Not tested":
 814             pass
 815         elif item[-4] == u"Not tested":
 816             item.append(u"New in CSIT-2001")
 817             item.append(u"New in CSIT-2001")
 818         # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
 819         #     item.append(u"See footnote [1]")
 820         #     footnote = True
 821         elif data_r_mean and data_c_mean:
 822             delta, d_stdev = relative_change_stdev(
 823                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
 824             )
 825             item.append(round(delta))
 826             item.append(round(d_stdev))
 827         if (len(item) == len(header)) and (item[-4] != u"Not tested"):
 828             tbl_lst.append(item)
 829
 830     tbl_lst = _tpc_sort_table(tbl_lst)
 831
 832     # Generate csv tables:
 833     csv_file = f"{table[u'output-file']}.csv"
 834     with open(csv_file, u"wt") as file_handler:
 835         file_handler.write(header_str)
 836         for test in tbl_lst:
 837             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
 838
 839     txt_file_name = f"{table[u'output-file']}.txt"
 840     convert_csv_to_pretty_txt(csv_file, txt_file_name)
 841
 842     if footnote:
 843         with open(txt_file_name, u'a') as txt_file:
 844             txt_file.writelines([
 845                 u"\nFootnotes:\n",
 846                 u"[1] CSIT-1908 changed test methodology of dot1q tests in "
 847                 u"2-node testbeds, dot1q encapsulation is now used on both "
 848                 u"links of SUT.\n",
 849                 u"    Previously dot1q was used only on a single link with the "
 850                 u"other link carrying untagged Ethernet frames. This changes "
 851                 u"results\n",
 852                 u"    in slightly lower throughput in CSIT-1908 for these "
 853                 u"tests. See release notes."
 854             ])
 855
 856     # Generate html table:
 857     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
 858
 859
 860 def table_perf_comparison_nic(table, input_data):
 861     """Generate the table(s) with algorithm: table_perf_comparison
 862     specified in the specification file.
 863
 864     :param table: Table to generate.
 865     :param input_data: Data to process.
 866     :type table: pandas.Series
 867     :type input_data: InputData
 868     """
 869
 870     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 871
 872     # Transform the data
 873     logging.info(
 874         f"    Creating the data set for the {table.get(u'type', u'')} "
 875         f"{table.get(u'title', u'')}."
 876     )
 877     data = input_data.filter_data(table, continue_on_error=True)
 878
 879     # Prepare the header of the tables
 880     try:
 881         header = [u"Test case", ]
 882
 883         if table[u"include-tests"] == u"MRR":
 884             hdr_param = u"Rec Rate"
 885         else:
 886             hdr_param = u"Thput"
 887
 888         history = table.get(u"history", list())
 889         for item in history:
 890             header.extend(
 891                 [
 892                     f"{item[u'title']} {hdr_param} [Mpps]",
 893                     f"{item[u'title']} Stdev [Mpps]"
 894                 ]
 895             )
 896         header.extend(
 897             [
 898                 f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
 899                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
 900                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
 901                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
 902                 u"Delta [%]",
 903                 u"Stdev of delta [%]"
 904             ]
 905         )
 906         header_str = u",".join(header) + u"\n"
 907     except (AttributeError, KeyError) as err:
 908         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 909         return
 910
 911     # Prepare data to the table:
 912     tbl_dict = dict()
 913     # topo = u""
 914     for job, builds in table[u"reference"][u"data"].items():
 915         # topo = u"2n-skx" if u"2n-skx" in job else u""
 916         for build in builds:
 917             for tst_name, tst_data in data[job][str(build)].items():
 918                 if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
 919                     continue
 920                 tst_name_mod = _tpc_modify_test_name(tst_name)
 921                 if (u"across topologies" in table[u"title"].lower() or
 922                         (u" 3n-" in table[u"title"].lower() and
 923                          u" 2n-" in table[u"title"].lower())):
 924                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 925                 if tbl_dict.get(tst_name_mod, None) is None:
 926                     name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 927                     if u"across testbeds" in table[u"title"].lower() or \
 928                             u"across topologies" in table[u"title"].lower():
 929                         name = _tpc_modify_displayed_test_name(name)
 930                     tbl_dict[tst_name_mod] = {
 931                         u"name": name,
 932                         u"ref-data": list(),
 933                         u"cmp-data": list()
 934                     }
 935                 _tpc_insert_data(
 936                     target=tbl_dict[tst_name_mod][u"ref-data"],
 937                     src=tst_data,
 938                     include_tests=table[u"include-tests"]
 939                 )
 940
 941     replacement = table[u"reference"].get(u"data-replacement", None)
 942     if replacement:
 943         create_new_list = True
 944         rpl_data = input_data.filter_data(
 945             table, data=replacement, continue_on_error=True)
 946         for job, builds in replacement.items():
 947             for build in builds:
 948                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 949                     if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
 950                         continue
 951                     tst_name_mod = _tpc_modify_test_name(tst_name)
 952                     if (u"across topologies" in table[u"title"].lower() or
 953                             (u" 3n-" in table[u"title"].lower() and
 954                              u" 2n-" in table[u"title"].lower())):
 955                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 956                     if tbl_dict.get(tst_name_mod, None) is None:
 957                         name = \
 958                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 959                         if u"across testbeds" in table[u"title"].lower() or \
 960                                 u"across topologies" in table[u"title"].lower():
 961                             name = _tpc_modify_displayed_test_name(name)
 962                         tbl_dict[tst_name_mod] = {
 963                             u"name": name,
 964                             u"ref-data": list(),
 965                             u"cmp-data": list()
 966                         }
 967                     if create_new_list:
 968                         create_new_list = False
 969                         tbl_dict[tst_name_mod][u"ref-data"] = list()
 970
 971                     _tpc_insert_data(
 972                         target=tbl_dict[tst_name_mod][u"ref-data"],
 973                         src=tst_data,
 974                         include_tests=table[u"include-tests"]
 975                     )
 976
 977     for job, builds in table[u"compare"][u"data"].items():
 978         for build in builds:
 979             for tst_name, tst_data in data[job][str(build)].items():
 980                 if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
 981                     continue
 982                 tst_name_mod = _tpc_modify_test_name(tst_name)
 983                 if (u"across topologies" in table[u"title"].lower() or
 984                         (u" 3n-" in table[u"title"].lower() and
 985                          u" 2n-" in table[u"title"].lower())):
 986                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 987                 if tbl_dict.get(tst_name_mod, None) is None:
 988                     name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 989                     if u"across testbeds" in table[u"title"].lower() or \
 990                             u"across topologies" in table[u"title"].lower():
 991                         name = _tpc_modify_displayed_test_name(name)
 992                     tbl_dict[tst_name_mod] = {
 993                         u"name": name,
 994                         u"ref-data": list(),
 995                         u"cmp-data": list()
 996                     }
 997                 _tpc_insert_data(
 998                     target=tbl_dict[tst_name_mod][u"cmp-data"],
 999                     src=tst_data,
1000                     include_tests=table[u"include-tests"]
1001                 )
1002
1003     replacement = table[u"compare"].get(u"data-replacement", None)
1004     if replacement:
1005         create_new_list = True
1006         rpl_data = input_data.filter_data(
1007             table, data=replacement, continue_on_error=True)
1008         for job, builds in replacement.items():
1009             for build in builds:
1010                 for tst_name, tst_data in rpl_data[job][str(build)].items():
1011                     if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
1012                         continue
1013                     tst_name_mod = _tpc_modify_test_name(tst_name)
1014                     if (u"across topologies" in table[u"title"].lower() or
1015                             (u" 3n-" in table[u"title"].lower() and
1016                              u" 2n-" in table[u"title"].lower())):
1017                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1018                     if tbl_dict.get(tst_name_mod, None) is None:
1019                         name = \
1020                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
1021                         if u"across testbeds" in table[u"title"].lower() or \
1022                                 u"across topologies" in table[u"title"].lower():
1023                             name = _tpc_modify_displayed_test_name(name)
1024                         tbl_dict[tst_name_mod] = {
1025                             u"name": name,
1026                             u"ref-data": list(),
1027                             u"cmp-data": list()
1028                         }
1029                     if create_new_list:
1030                         create_new_list = False
1031                         tbl_dict[tst_name_mod][u"cmp-data"] = list()
1032
1033                     _tpc_insert_data(
1034                         target=tbl_dict[tst_name_mod][u"cmp-data"],
1035                         src=tst_data,
1036                         include_tests=table[u"include-tests"]
1037                     )
1038
1039     for item in history:
1040         for job, builds in item[u"data"].items():
1041             for build in builds:
1042                 for tst_name, tst_data in data[job][str(build)].items():
1043                     if item[u"nic"] not in tst_data[u"tags"]:
1044                         continue
1045                     tst_name_mod = _tpc_modify_test_name(tst_name)
1046                     if (u"across topologies" in table[u"title"].lower() or
1047                             (u" 3n-" in table[u"title"].lower() and
1048                              u" 2n-" in table[u"title"].lower())):
1049                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1050                     if tbl_dict.get(tst_name_mod, None) is None:
1051                         continue
1052                     if tbl_dict[tst_name_mod].get(u"history", None) is None:
1053                         tbl_dict[tst_name_mod][u"history"] = OrderedDict()
1054                     if tbl_dict[tst_name_mod][u"history"].\
1055                             get(item[u"title"], None) is None:
1056                         tbl_dict[tst_name_mod][u"history"][item[
1057                             u"title"]] = list()
1058                     try:
1059                         if table[u"include-tests"] == u"MRR":
1060                             res = tst_data[u"result"][u"receive-rate"]
1061                         elif table[u"include-tests"] == u"PDR":
1062                             res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
1063                         elif table[u"include-tests"] == u"NDR":
1064                             res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
1065                         else:
1066                             continue
1067                         tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
1068                             append(res)
1069                     except (TypeError, KeyError):
1070                         pass
1071
1072     tbl_lst = list()
1073     footnote = False
1074     for tst_name in tbl_dict:
1075         item = [tbl_dict[tst_name][u"name"], ]
1076         if history:
1077             if tbl_dict[tst_name].get(u"history", None) is not None:
1078                 for hist_data in tbl_dict[tst_name][u"history"].values():
1079                     if hist_data:
1080                         item.append(round(mean(hist_data) / 1000000, 2))
1081                         item.append(round(stdev(hist_data) / 1000000, 2))
1082                     else:
1083                         item.extend([u"Not tested", u"Not tested"])
1084             else:
1085                 item.extend([u"Not tested", u"Not tested"])
1086         data_r = tbl_dict[tst_name][u"ref-data"]
1087         if data_r:
1088             data_r_mean = mean(data_r)
1089             item.append(round(data_r_mean / 1000000, 2))
1090             data_r_stdev = stdev(data_r)
1091             item.append(round(data_r_stdev / 1000000, 2))
1092         else:
1093             data_r_mean = None
1094             data_r_stdev = None
1095             item.extend([u"Not tested", u"Not tested"])
1096         data_c = tbl_dict[tst_name][u"cmp-data"]
1097         if data_c:
1098             data_c_mean = mean(data_c)
1099             item.append(round(data_c_mean / 1000000, 2))
1100             data_c_stdev = stdev(data_c)
1101             item.append(round(data_c_stdev / 1000000, 2))
1102         else:
1103             data_c_mean = None
1104             data_c_stdev = None
1105             item.extend([u"Not tested", u"Not tested"])
1106         if item[-2] == u"Not tested":
1107             pass
1108         elif item[-4] == u"Not tested":
1109             item.append(u"New in CSIT-2001")
1110             item.append(u"New in CSIT-2001")
1111         # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
1112         #     item.append(u"See footnote [1]")
1113         #     footnote = True
1114         elif data_r_mean and data_c_mean:
1115             delta, d_stdev = relative_change_stdev(
1116                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
1117             )
1118             item.append(round(delta))
1119             item.append(round(d_stdev))
1120         if (len(item) == len(header)) and (item[-4] != u"Not tested"):
1121             tbl_lst.append(item)
1122
1123     tbl_lst = _tpc_sort_table(tbl_lst)
1124
1125     # Generate csv tables:
1126     csv_file = f"{table[u'output-file']}.csv"
1127     with open(csv_file, u"wt") as file_handler:
1128         file_handler.write(header_str)
1129         for test in tbl_lst:
1130             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
1131
1132     txt_file_name = f"{table[u'output-file']}.txt"
1133     convert_csv_to_pretty_txt(csv_file, txt_file_name)
1134
1135     if footnote:
1136         with open(txt_file_name, u'a') as txt_file:
1137             txt_file.writelines([
1138                 u"\nFootnotes:\n",
1139                 u"[1] CSIT-1908 changed test methodology of dot1q tests in "
1140                 u"2-node testbeds, dot1q encapsulation is now used on both "
1141                 u"links of SUT.\n",
1142                 u"    Previously dot1q was used only on a single link with the "
1143                 u"other link carrying untagged Ethernet frames. This changes "
1144                 u"results\n",
1145                 u"    in slightly lower throughput in CSIT-1908 for these "
1146                 u"tests. See release notes."
1147             ])
1148
1149     # Generate html table:
1150     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
1151
1152
1153 def table_nics_comparison(table, input_data):
1154     """Generate the table(s) with algorithm: table_nics_comparison
1155     specified in the specification file.
1156
1157     :param table: Table to generate.
1158     :param input_data: Data to process.
1159     :type table: pandas.Series
1160     :type input_data: InputData
1161     """
1162
1163     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1164
1165     # Transform the data
1166     logging.info(
1167         f"    Creating the data set for the {table.get(u'type', u'')} "
1168         f"{table.get(u'title', u'')}."
1169     )
1170     data = input_data.filter_data(table, continue_on_error=True)
1171
1172     # Prepare the header of the tables
1173     try:
1174         header = [u"Test case", ]
1175
1176         if table[u"include-tests"] == u"MRR":
1177             hdr_param = u"Rec Rate"
1178         else:
1179             hdr_param = u"Thput"
1180
1181         header.extend(
1182             [
1183                 f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
1184                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
1185                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
1186                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
1187                 u"Delta [%]",
1188                 u"Stdev of delta [%]"
1189             ]
1190         )
1191
1192     except (AttributeError, KeyError) as err:
1193         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1194         return
1195
1196     # Prepare data to the table:
1197     tbl_dict = dict()
1198     for job, builds in table[u"data"].items():
1199         for build in builds:
1200             for tst_name, tst_data in data[job][str(build)].items():
1201                 tst_name_mod = _tpc_modify_test_name(tst_name)
1202                 if tbl_dict.get(tst_name_mod, None) is None:
1203                     name = u"-".join(tst_data[u"name"].split(u"-")[:-1])
1204                     tbl_dict[tst_name_mod] = {
1205                         u"name": name,
1206                         u"ref-data": list(),
1207                         u"cmp-data": list()
1208                     }
1209                 try:
1210                     if table[u"include-tests"] == u"MRR":
1211                         result = tst_data[u"result"][u"receive-rate"]
1212                     elif table[u"include-tests"] == u"PDR":
1213                         result = tst_data[u"throughput"][u"PDR"][u"LOWER"]
1214                     elif table[u"include-tests"] == u"NDR":
1215                         result = tst_data[u"throughput"][u"NDR"][u"LOWER"]
1216                     else:
1217                         continue
1218
1219                     if result and \
1220                             table[u"reference"][u"nic"] in tst_data[u"tags"]:
1221                         tbl_dict[tst_name_mod][u"ref-data"].append(result)
1222                     elif result and \
1223                             table[u"compare"][u"nic"] in tst_data[u"tags"]:
1224                         tbl_dict[tst_name_mod][u"cmp-data"].append(result)
1225                 except (TypeError, KeyError) as err:
1226                     logging.debug(f"No data for {tst_name}\n{repr(err)}")
1227                     # No data in output.xml for this test
1228
1229     tbl_lst = list()
1230     for tst_name in tbl_dict:
1231         item = [tbl_dict[tst_name][u"name"], ]
1232         data_r = tbl_dict[tst_name][u"ref-data"]
1233         if data_r:
1234             data_r_mean = mean(data_r)
1235             item.append(round(data_r_mean / 1000000, 2))
1236             data_r_stdev = stdev(data_r)
1237             item.append(round(data_r_stdev / 1000000, 2))
1238         else:
1239             data_r_mean = None
1240             data_r_stdev = None
1241             item.extend([None, None])
1242         data_c = tbl_dict[tst_name][u"cmp-data"]
1243         if data_c:
1244             data_c_mean = mean(data_c)
1245             item.append(round(data_c_mean / 1000000, 2))
1246             data_c_stdev = stdev(data_c)
1247             item.append(round(data_c_stdev / 1000000, 2))
1248         else:
1249             data_c_mean = None
1250             data_c_stdev = None
1251             item.extend([None, None])
1252         if data_r_mean and data_c_mean:
1253             delta, d_stdev = relative_change_stdev(
1254                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
1255             )
1256             item.append(round(delta))
1257             item.append(round(d_stdev))
1258             tbl_lst.append(item)
1259
1260     # Sort the table according to the relative change
1261     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1262
1263     # Generate csv tables:
1264     with open(f"{table[u'output-file']}.csv", u"wt") as file_handler:
1265         file_handler.write(u",".join(header) + u"\n")
1266         for test in tbl_lst:
1267             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
1268
1269     convert_csv_to_pretty_txt(f"{table[u'output-file']}.csv",
1270                               f"{table[u'output-file']}.txt")
1271
1272     # Generate html table:
1273     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
1274
1275
1276 def table_soak_vs_ndr(table, input_data):
1277     """Generate the table(s) with algorithm: table_soak_vs_ndr
1278     specified in the specification file.
1279
1280     :param table: Table to generate.
1281     :param input_data: Data to process.
1282     :type table: pandas.Series
1283     :type input_data: InputData
1284     """
1285
1286     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1287
1288     # Transform the data
1289     logging.info(
1290         f"    Creating the data set for the {table.get(u'type', u'')} "
1291         f"{table.get(u'title', u'')}."
1292     )
1293     data = input_data.filter_data(table, continue_on_error=True)
1294
1295     # Prepare the header of the table
1296     try:
1297         header = [
1298             u"Test case",
1299             f"{table[u'reference'][u'title']} Thput [Mpps]",
1300             f"{table[u'reference'][u'title']} Stdev [Mpps]",
1301             f"{table[u'compare'][u'title']} Thput [Mpps]",
1302             f"{table[u'compare'][u'title']} Stdev [Mpps]",
1303             u"Delta [%]",
1304             u"Stdev of delta [%]"
1305         ]
1306         header_str = u",".join(header) + u"\n"
1307     except (AttributeError, KeyError) as err:
1308         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1309         return
1310
1311     # Create a list of available SOAK test results:
1312     tbl_dict = dict()
1313     for job, builds in table[u"compare"][u"data"].items():
1314         for build in builds:
1315             for tst_name, tst_data in data[job][str(build)].items():
1316                 if tst_data[u"type"] == u"SOAK":
1317                     tst_name_mod = tst_name.replace(u"-soak", u"")
1318                     if tbl_dict.get(tst_name_mod, None) is None:
1319                         groups = re.search(REGEX_NIC, tst_data[u"parent"])
1320                         nic = groups.group(0) if groups else u""
1321                         name = (
1322                             f"{nic}-"
1323                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
1324                         )
1325                         tbl_dict[tst_name_mod] = {
1326                             u"name": name,
1327                             u"ref-data": list(),
1328                             u"cmp-data": list()
1329                         }
1330                     try:
1331                         tbl_dict[tst_name_mod][u"cmp-data"].append(
1332                             tst_data[u"throughput"][u"LOWER"])
1333                     except (KeyError, TypeError):
1334                         pass
1335     tests_lst = tbl_dict.keys()
1336
1337     # Add corresponding NDR test results:
1338     for job, builds in table[u"reference"][u"data"].items():
1339         for build in builds:
1340             for tst_name, tst_data in data[job][str(build)].items():
1341                 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
1342                     replace(u"-mrr", u"")
1343                 if tst_name_mod not in tests_lst:
1344                     continue
1345                 try:
1346                     if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
1347                         continue
1348                     if table[u"include-tests"] == u"MRR":
1349                         result = tst_data[u"result"][u"receive-rate"]
1350                     elif table[u"include-tests"] == u"PDR":
1351                         result = \
1352                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
1353                     elif table[u"include-tests"] == u"NDR":
1354                         result = \
1355                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
1356                     else:
1357                         result = None
1358                     if result is not None:
1359                         tbl_dict[tst_name_mod][u"ref-data"].append(
1360                             result)
1361                 except (KeyError, TypeError):
1362                     continue
1363
1364     tbl_lst = list()
1365     for tst_name in tbl_dict:
1366         item = [tbl_dict[tst_name][u"name"], ]
1367         data_r = tbl_dict[tst_name][u"ref-data"]
1368         if data_r:
1369             data_r_mean = mean(data_r)
1370             item.append(round(data_r_mean / 1000000, 2))
1371             data_r_stdev = stdev(data_r)
1372             item.append(round(data_r_stdev / 1000000, 2))
1373         else:
1374             data_r_mean = None
1375             data_r_stdev = None
1376             item.extend([None, None])
1377         data_c = tbl_dict[tst_name][u"cmp-data"]
1378         if data_c:
1379             data_c_mean = mean(data_c)
1380             item.append(round(data_c_mean / 1000000, 2))
1381             data_c_stdev = stdev(data_c)
1382             item.append(round(data_c_stdev / 1000000, 2))
1383         else:
1384             data_c_mean = None
1385             data_c_stdev = None
1386             item.extend([None, None])
1387         if data_r_mean and data_c_mean:
1388             delta, d_stdev = relative_change_stdev(
1389                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
1390             item.append(round(delta))
1391             item.append(round(d_stdev))
1392             tbl_lst.append(item)
1393
1394     # Sort the table according to the relative change
1395     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1396
1397     # Generate csv tables:
1398     csv_file = f"{table[u'output-file']}.csv"
1399     with open(csv_file, u"wt") as file_handler:
1400         file_handler.write(header_str)
1401         for test in tbl_lst:
1402             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
1403
1404     convert_csv_to_pretty_txt(csv_file, f"{table[u'output-file']}.txt")
1405
1406     # Generate html table:
1407     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
1408
1409
1410 def table_perf_trending_dash(table, input_data):
1411     """Generate the table(s) with algorithm:
1412     table_perf_trending_dash
1413     specified in the specification file.
1414
1415     :param table: Table to generate.
1416     :param input_data: Data to process.
1417     :type table: pandas.Series
1418     :type input_data: InputData
1419     """
1420
1421     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1422
1423     # Transform the data
1424     logging.info(
1425         f"    Creating the data set for the {table.get(u'type', u'')} "
1426         f"{table.get(u'title', u'')}."
1427     )
1428     data = input_data.filter_data(table, continue_on_error=True)
1429
1430     # Prepare the header of the tables
1431     header = [
1432         u"Test Case",
1433         u"Trend [Mpps]",
1434         u"Short-Term Change [%]",
1435         u"Long-Term Change [%]",
1436         u"Regressions [#]",
1437         u"Progressions [#]"
1438     ]
1439     header_str = u",".join(header) + u"\n"
1440
1441     # Prepare data to the table:
1442     tbl_dict = dict()
1443     for job, builds in table[u"data"].items():
1444         for build in builds:
1445             for tst_name, tst_data in data[job][str(build)].items():
1446                 if tst_name.lower() in table.get(u"ignore-list", list()):
1447                     continue
1448                 if tbl_dict.get(tst_name, None) is None:
1449                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1450                     if not groups:
1451                         continue
1452                     nic = groups.group(0)
1453                     tbl_dict[tst_name] = {
1454                         u"name": f"{nic}-{tst_data[u'name']}",
1455                         u"data": OrderedDict()
1456                     }
1457                 try:
1458                     tbl_dict[tst_name][u"data"][str(build)] = \
1459                         tst_data[u"result"][u"receive-rate"]
1460                 except (TypeError, KeyError):
1461                     pass  # No data in output.xml for this test
1462
1463     tbl_lst = list()
1464     for tst_name in tbl_dict:
1465         data_t = tbl_dict[tst_name][u"data"]
1466         if len(data_t) < 2:
1467             continue
1468
1469         classification_lst, avgs = classify_anomalies(data_t)
1470
1471         win_size = min(len(data_t), table[u"window"])
1472         long_win_size = min(len(data_t), table[u"long-trend-window"])
1473
1474         try:
1475             max_long_avg = max(
1476                 [x for x in avgs[-long_win_size:-win_size]
1477                  if not isnan(x)])
1478         except ValueError:
1479             max_long_avg = nan
1480         last_avg = avgs[-1]
1481         avg_week_ago = avgs[max(-win_size, -len(avgs))]
1482
1483         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
1484             rel_change_last = nan
1485         else:
1486             rel_change_last = round(
1487                 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
1488
1489         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
1490             rel_change_long = nan
1491         else:
1492             rel_change_long = round(
1493                 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
1494
1495         if classification_lst:
1496             if isnan(rel_change_last) and isnan(rel_change_long):
1497                 continue
1498             if isnan(last_avg) or isnan(rel_change_last) or \
1499                     isnan(rel_change_long):
1500                 continue
1501             tbl_lst.append(
1502                 [tbl_dict[tst_name][u"name"],
1503                  round(last_avg / 1000000, 2),
1504                  rel_change_last,
1505                  rel_change_long,
1506                  classification_lst[-win_size:].count(u"regression"),
1507                  classification_lst[-win_size:].count(u"progression")])
1508
1509     tbl_lst.sort(key=lambda rel: rel[0])
1510
1511     tbl_sorted = list()
1512     for nrr in range(table[u"window"], -1, -1):
1513         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
1514         for nrp in range(table[u"window"], -1, -1):
1515             tbl_out = [item for item in tbl_reg if item[5] == nrp]
1516             tbl_out.sort(key=lambda rel: rel[2])
1517             tbl_sorted.extend(tbl_out)
1518
1519     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1520
1521     logging.info(f"    Writing file: {file_name}")
1522     with open(file_name, u"wt") as file_handler:
1523         file_handler.write(header_str)
1524         for test in tbl_sorted:
1525             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1526
1527     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1528     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1529
1530
1531 def _generate_url(testbed, test_name):
1532     """Generate URL to a trending plot from the name of the test case.
1533
1534     :param testbed: The testbed used for testing.
1535     :param test_name: The name of the test case.
1536     :type testbed: str
1537     :type test_name: str
1538     :returns: The URL to the plot with the trending data for the given test
1539         case.
1540     :rtype str
1541     """
1542
1543     if u"x520" in test_name:
1544         nic = u"x520"
1545     elif u"x710" in test_name:
1546         nic = u"x710"
1547     elif u"xl710" in test_name:
1548         nic = u"xl710"
1549     elif u"xxv710" in test_name:
1550         nic = u"xxv710"
1551     elif u"vic1227" in test_name:
1552         nic = u"vic1227"
1553     elif u"vic1385" in test_name:
1554         nic = u"vic1385"
1555     elif u"x553" in test_name:
1556         nic = u"x553"
1557     elif u"cx556" in test_name or u"cx556a" in test_name:
1558         nic = u"cx556a"
1559     else:
1560         nic = u""
1561
1562     if u"64b" in test_name:
1563         frame_size = u"64b"
1564     elif u"78b" in test_name:
1565         frame_size = u"78b"
1566     elif u"imix" in test_name:
1567         frame_size = u"imix"
1568     elif u"9000b" in test_name:
1569         frame_size = u"9000b"
1570     elif u"1518b" in test_name:
1571         frame_size = u"1518b"
1572     elif u"114b" in test_name:
1573         frame_size = u"114b"
1574     else:
1575         frame_size = u""
1576
1577     if u"1t1c" in test_name or \
1578         (u"-1c-" in test_name and
1579          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1580         cores = u"1t1c"
1581     elif u"2t2c" in test_name or \
1582          (u"-2c-" in test_name and
1583           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1584         cores = u"2t2c"
1585     elif u"4t4c" in test_name or \
1586          (u"-4c-" in test_name and
1587           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1588         cores = u"4t4c"
1589     elif u"2t1c" in test_name or \
1590          (u"-1c-" in test_name and
1591           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1592         cores = u"2t1c"
1593     elif u"4t2c" in test_name or \
1594          (u"-2c-" in test_name and
1595           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1596         cores = u"4t2c"
1597     elif u"8t4c" in test_name or \
1598          (u"-4c-" in test_name and
1599           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1600         cores = u"8t4c"
1601     else:
1602         cores = u""
1603
1604     if u"testpmd" in test_name:
1605         driver = u"testpmd"
1606     elif u"l3fwd" in test_name:
1607         driver = u"l3fwd"
1608     elif u"avf" in test_name:
1609         driver = u"avf"
1610     elif u"rdma" in test_name:
1611         driver = u"rdma"
1612     elif u"dnv" in testbed or u"tsh" in testbed:
1613         driver = u"ixgbe"
1614     else:
1615         driver = u"dpdk"
1616
1617     if u"acl" in test_name or \
1618             u"macip" in test_name or \
1619             u"nat" in test_name or \
1620             u"policer" in test_name or \
1621             u"cop" in test_name:
1622         bsf = u"features"
1623     elif u"scale" in test_name:
1624         bsf = u"scale"
1625     elif u"base" in test_name:
1626         bsf = u"base"
1627     else:
1628         bsf = u"base"
1629
1630     if u"114b" in test_name and u"vhost" in test_name:
1631         domain = u"vts"
1632     elif u"testpmd" in test_name or u"l3fwd" in test_name:
1633         domain = u"dpdk"
1634     elif u"memif" in test_name:
1635         domain = u"container_memif"
1636     elif u"srv6" in test_name:
1637         domain = u"srv6"
1638     elif u"vhost" in test_name:
1639         domain = u"vhost"
1640         if u"vppl2xc" in test_name:
1641             driver += u"-vpp"
1642         else:
1643             driver += u"-testpmd"
1644         if u"lbvpplacp" in test_name:
1645             bsf += u"-link-bonding"
1646     elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1647         domain = u"nf_service_density_vnfc"
1648     elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1649         domain = u"nf_service_density_cnfc"
1650     elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1651         domain = u"nf_service_density_cnfp"
1652     elif u"ipsec" in test_name:
1653         domain = u"ipsec"
1654         if u"sw" in test_name:
1655             bsf += u"-sw"
1656         elif u"hw" in test_name:
1657             bsf += u"-hw"
1658     elif u"ethip4vxlan" in test_name:
1659         domain = u"ip4_tunnels"
1660     elif u"ip4base" in test_name or u"ip4scale" in test_name:
1661         domain = u"ip4"
1662     elif u"ip6base" in test_name or u"ip6scale" in test_name:
1663         domain = u"ip6"
1664     elif u"l2xcbase" in test_name or \
1665             u"l2xcscale" in test_name or \
1666             u"l2bdbasemaclrn" in test_name or \
1667             u"l2bdscale" in test_name or \
1668             u"l2patch" in test_name:
1669         domain = u"l2"
1670     else:
1671         domain = u""
1672
1673     file_name = u"-".join((domain, testbed, nic)) + u".html#"
1674     anchor_name = u"-".join((frame_size, cores, bsf, driver))
1675
1676     return file_name + anchor_name
1677
1678
1679 def table_perf_trending_dash_html(table, input_data):
1680     """Generate the table(s) with algorithm:
1681     table_perf_trending_dash_html specified in the specification
1682     file.
1683
1684     :param table: Table to generate.
1685     :param input_data: Data to process.
1686     :type table: dict
1687     :type input_data: InputData
1688     """
1689
1690     _ = input_data
1691
1692     if not table.get(u"testbed", None):
1693         logging.error(
1694             f"The testbed is not defined for the table "
1695             f"{table.get(u'title', u'')}."
1696         )
1697         return
1698
1699     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1700
1701     try:
1702         with open(table[u"input-file"], u'rt') as csv_file:
1703             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1704     except KeyError:
1705         logging.warning(u"The input file is not defined.")
1706         return
1707     except csv.Error as err:
1708         logging.warning(
1709             f"Not possible to process the file {table[u'input-file']}.\n"
1710             f"{repr(err)}"
1711         )
1712         return
1713
1714     # Table:
1715     dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1716
1717     # Table header:
1718     trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1719     for idx, item in enumerate(csv_lst[0]):
1720         alignment = u"left" if idx == 0 else u"center"
1721         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1722         thead.text = item
1723
1724     # Rows:
1725     colors = {
1726         u"regression": (
1727             u"#ffcccc",
1728             u"#ff9999"
1729         ),
1730         u"progression": (
1731             u"#c6ecc6",
1732             u"#9fdf9f"
1733         ),
1734         u"normal": (
1735             u"#e9f1fb",
1736             u"#d4e4f7"
1737         )
1738     }
1739     for r_idx, row in enumerate(csv_lst[1:]):
1740         if int(row[4]):
1741             color = u"regression"
1742         elif int(row[5]):
1743             color = u"progression"
1744         else:
1745             color = u"normal"
1746         trow = ET.SubElement(
1747             dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
1748         )
1749
1750         # Columns:
1751         for c_idx, item in enumerate(row):
1752             tdata = ET.SubElement(
1753                 trow,
1754                 u"td",
1755                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1756             )
1757             # Name:
1758             if c_idx == 0:
1759                 ref = ET.SubElement(
1760                     tdata,
1761                     u"a",
1762                     attrib=dict(
1763                         href=f"../trending/"
1764                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1765                     )
1766                 )
1767                 ref.text = item
1768             else:
1769                 tdata.text = item
1770     try:
1771         with open(table[u"output-file"], u'w') as html_file:
1772             logging.info(f"    Writing file: {table[u'output-file']}")
1773             html_file.write(u".. raw:: html\n\n\t")
1774             html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
1775             html_file.write(u"\n\t<p><br><br></p>\n")
1776     except KeyError:
1777         logging.warning(u"The output file is not defined.")
1778         return
1779
1780
1781 def table_last_failed_tests(table, input_data):
1782     """Generate the table(s) with algorithm: table_last_failed_tests
1783     specified in the specification file.
1784
1785     :param table: Table to generate.
1786     :param input_data: Data to process.
1787     :type table: pandas.Series
1788     :type input_data: InputData
1789     """
1790
1791     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1792
1793     # Transform the data
1794     logging.info(
1795         f"    Creating the data set for the {table.get(u'type', u'')} "
1796         f"{table.get(u'title', u'')}."
1797     )
1798
1799     data = input_data.filter_data(table, continue_on_error=True)
1800
1801     if data is None or data.empty:
1802         logging.warning(
1803             f"    No data for the {table.get(u'type', u'')} "
1804             f"{table.get(u'title', u'')}."
1805         )
1806         return
1807
1808     tbl_list = list()
1809     for job, builds in table[u"data"].items():
1810         for build in builds:
1811             build = str(build)
1812             try:
1813                 version = input_data.metadata(job, build).get(u"version", u"")
1814             except KeyError:
1815                 logging.error(f"Data for {job}: {build} is not present.")
1816                 return
1817             tbl_list.append(build)
1818             tbl_list.append(version)
1819             failed_tests = list()
1820             passed = 0
1821             failed = 0
1822             for tst_data in data[job][build].values:
1823                 if tst_data[u"status"] != u"FAIL":
1824                     passed += 1
1825                     continue
1826                 failed += 1
1827                 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1828                 if not groups:
1829                     continue
1830                 nic = groups.group(0)
1831                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
1832             tbl_list.append(str(passed))
1833             tbl_list.append(str(failed))
1834             tbl_list.extend(failed_tests)
1835
1836     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1837     logging.info(f"    Writing file: {file_name}")
1838     with open(file_name, u"wt") as file_handler:
1839         for test in tbl_list:
1840             file_handler.write(test + u'\n')
1841
1842
1843 def table_failed_tests(table, input_data):
1844     """Generate the table(s) with algorithm: table_failed_tests
1845     specified in the specification file.
1846
1847     :param table: Table to generate.
1848     :param input_data: Data to process.
1849     :type table: pandas.Series
1850     :type input_data: InputData
1851     """
1852
1853     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1854
1855     # Transform the data
1856     logging.info(
1857         f"    Creating the data set for the {table.get(u'type', u'')} "
1858         f"{table.get(u'title', u'')}."
1859     )
1860     data = input_data.filter_data(table, continue_on_error=True)
1861
1862     # Prepare the header of the tables
1863     header = [
1864         u"Test Case",
1865         u"Failures [#]",
1866         u"Last Failure [Time]",
1867         u"Last Failure [VPP-Build-Id]",
1868         u"Last Failure [CSIT-Job-Build-Id]"
1869     ]
1870
1871     # Generate the data for the table according to the model in the table
1872     # specification
1873
1874     now = dt.utcnow()
1875     timeperiod = timedelta(int(table.get(u"window", 7)))
1876
1877     tbl_dict = dict()
1878     for job, builds in table[u"data"].items():
1879         for build in builds:
1880             build = str(build)
1881             for tst_name, tst_data in data[job][build].items():
1882                 if tst_name.lower() in table.get(u"ignore-list", list()):
1883                     continue
1884                 if tbl_dict.get(tst_name, None) is None:
1885                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1886                     if not groups:
1887                         continue
1888                     nic = groups.group(0)
1889                     tbl_dict[tst_name] = {
1890                         u"name": f"{nic}-{tst_data[u'name']}",
1891                         u"data": OrderedDict()
1892                     }
1893                 try:
1894                     generated = input_data.metadata(job, build).\
1895                         get(u"generated", u"")
1896                     if not generated:
1897                         continue
1898                     then = dt.strptime(generated, u"%Y%m%d %H:%M")
1899                     if (now - then) <= timeperiod:
1900                         tbl_dict[tst_name][u"data"][build] = (
1901                             tst_data[u"status"],
1902                             generated,
1903                             input_data.metadata(job, build).get(u"version",
1904                                                                 u""),
1905                             build
1906                         )
1907                 except (TypeError, KeyError) as err:
1908                     logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
1909
1910     max_fails = 0
1911     tbl_lst = list()
1912     for tst_data in tbl_dict.values():
1913         fails_nr = 0
1914         fails_last_date = u""
1915         fails_last_vpp = u""
1916         fails_last_csit = u""
1917         for val in tst_data[u"data"].values():
1918             if val[0] == u"FAIL":
1919                 fails_nr += 1
1920                 fails_last_date = val[1]
1921                 fails_last_vpp = val[2]
1922                 fails_last_csit = val[3]
1923         if fails_nr:
1924             max_fails = fails_nr if fails_nr > max_fails else max_fails
1925             tbl_lst.append(
1926                 [
1927                     tst_data[u"name"],
1928                     fails_nr,
1929                     fails_last_date,
1930                     fails_last_vpp,
1931                     f"mrr-daily-build-{fails_last_csit}"
1932                 ]
1933             )
1934
1935     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1936     tbl_sorted = list()
1937     for nrf in range(max_fails, -1, -1):
1938         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1939         tbl_sorted.extend(tbl_fails)
1940
1941     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1942     logging.info(f"    Writing file: {file_name}")
1943     with open(file_name, u"wt") as file_handler:
1944         file_handler.write(u",".join(header) + u"\n")
1945         for test in tbl_sorted:
1946             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1947
1948     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1949     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1950
1951
1952 def table_failed_tests_html(table, input_data):
1953     """Generate the table(s) with algorithm: table_failed_tests_html
1954     specified in the specification file.
1955
1956     :param table: Table to generate.
1957     :param input_data: Data to process.
1958     :type table: pandas.Series
1959     :type input_data: InputData
1960     """
1961
1962     _ = input_data
1963
1964     if not table.get(u"testbed", None):
1965         logging.error(
1966             f"The testbed is not defined for the table "
1967             f"{table.get(u'title', u'')}."
1968         )
1969         return
1970
1971     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1972
1973     try:
1974         with open(table[u"input-file"], u'rt') as csv_file:
1975             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1976     except KeyError:
1977         logging.warning(u"The input file is not defined.")
1978         return
1979     except csv.Error as err:
1980         logging.warning(
1981             f"Not possible to process the file {table[u'input-file']}.\n"
1982             f"{repr(err)}"
1983         )
1984         return
1985
1986     # Table:
1987     failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1988
1989     # Table header:
1990     trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1991     for idx, item in enumerate(csv_lst[0]):
1992         alignment = u"left" if idx == 0 else u"center"
1993         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1994         thead.text = item
1995
1996     # Rows:
1997     colors = (u"#e9f1fb", u"#d4e4f7")
1998     for r_idx, row in enumerate(csv_lst[1:]):
1999         background = colors[r_idx % 2]
2000         trow = ET.SubElement(
2001             failed_tests, u"tr", attrib=dict(bgcolor=background)
2002         )
2003
2004         # Columns:
2005         for c_idx, item in enumerate(row):
2006             tdata = ET.SubElement(
2007                 trow,
2008                 u"td",
2009                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
2010             )
2011             # Name:
2012             if c_idx == 0:
2013                 ref = ET.SubElement(
2014                     tdata,
2015                     u"a",
2016                     attrib=dict(
2017                         href=f"../trending/"
2018                              f"{_generate_url(table.get(u'testbed', ''), item)}"
2019                     )
2020                 )
2021                 ref.text = item
2022             else:
2023                 tdata.text = item
2024     try:
2025         with open(table[u"output-file"], u'w') as html_file:
2026             logging.info(f"    Writing file: {table[u'output-file']}")
2027             html_file.write(u".. raw:: html\n\n\t")
2028             html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
2029             html_file.write(u"\n\t<p><br><br></p>\n")
2030     except KeyError:
2031         logging.warning(u"The output file is not defined.")
2032         return