resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from collections import OrderedDict
  23 from xml.etree import ElementTree as ET
  24 from datetime import datetime as dt
  25 from datetime import timedelta
  26
  27 import plotly.graph_objects as go
  28 import plotly.offline as ploff
  29 import pandas as pd
  30
  31 from numpy import nan, isnan
  32
  33 from pal_utils import mean, stdev, relative_change, classify_anomalies, \
  34     convert_csv_to_pretty_txt, relative_change_stdev
  35
  36
  37 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
  38
  39
  40 def generate_tables(spec, data):
  41     """Generate all tables specified in the specification file.
  42
  43     :param spec: Specification read from the specification file.
  44     :param data: Data to process.
  45     :type spec: Specification
  46     :type data: InputData
  47     """
  48
  49     generator = {
  50         u"table_details": table_details,
  51         u"table_merged_details": table_merged_details,
  52         u"table_perf_comparison": table_perf_comparison,
  53         u"table_perf_comparison_nic": table_perf_comparison_nic,
  54         u"table_nics_comparison": table_nics_comparison,
  55         u"table_soak_vs_ndr": table_soak_vs_ndr,
  56         u"table_perf_trending_dash": table_perf_trending_dash,
  57         u"table_perf_trending_dash_html": table_perf_trending_dash_html,
  58         u"table_last_failed_tests": table_last_failed_tests,
  59         u"table_failed_tests": table_failed_tests,
  60         u"table_failed_tests_html": table_failed_tests_html,
  61         u"table_oper_data_html": table_oper_data_html
  62     }
  63
  64     logging.info(u"Generating the tables ...")
  65     for table in spec.tables:
  66         try:
  67             generator[table[u"algorithm"]](table, data)
  68         except NameError as err:
  69             logging.error(
  70                 f"Probably algorithm {table[u'algorithm']} is not defined: "
  71                 f"{repr(err)}"
  72             )
  73     logging.info(u"Done.")
  74
  75
  76 def table_oper_data_html(table, input_data):
  77     """Generate the table(s) with algorithm: html_table_oper_data
  78     specified in the specification file.
  79
  80     :param table: Table to generate.
  81     :param input_data: Data to process.
  82     :type table: pandas.Series
  83     :type input_data: InputData
  84     """
  85
  86     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
  87     # Transform the data
  88     logging.info(
  89         f"    Creating the data set for the {table.get(u'type', u'')} "
  90         f"{table.get(u'title', u'')}."
  91     )
  92     data = input_data.filter_data(
  93         table,
  94         params=[u"name", u"parent", u"show-run", u"type"],
  95         continue_on_error=True
  96     )
  97     if data.empty:
  98         return
  99     data = input_data.merge_data(data)
 100     data.sort_index(inplace=True)
 101
 102     suites = input_data.filter_data(
 103         table,
 104         continue_on_error=True,
 105         data_set=u"suites"
 106     )
 107     if suites.empty:
 108         return
 109     suites = input_data.merge_data(suites)
 110
 111     def _generate_html_table(tst_data):
 112         """Generate an HTML table with operational data for the given test.
 113
 114         :param tst_data: Test data to be used to generate the table.
 115         :type tst_data: pandas.Series
 116         :returns: HTML table with operational data.
 117         :rtype: str
 118         """
 119
 120         colors = {
 121             u"header": u"#7eade7",
 122             u"empty": u"#ffffff",
 123             u"body": (u"#e9f1fb", u"#d4e4f7")
 124         }
 125
 126         tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
 127
 128         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
 129         thead = ET.SubElement(
 130             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 131         )
 132         thead.text = tst_data[u"name"]
 133
 134         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 135         thead = ET.SubElement(
 136             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 137         )
 138         thead.text = u"\t"
 139
 140         if tst_data.get(u"show-run", u"No Data") == u"No Data":
 141             trow = ET.SubElement(
 142                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 143             )
 144             tcol = ET.SubElement(
 145                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 146             )
 147             tcol.text = u"No Data"
 148             return str(ET.tostring(tbl, encoding=u"unicode"))
 149
 150         tbl_hdr = (
 151             u"Name",
 152             u"Nr of Vectors",
 153             u"Nr of Packets",
 154             u"Suspends",
 155             u"Cycles per Packet",
 156             u"Average Vector Size"
 157         )
 158
 159         for dut_name, dut_data in tst_data[u"show-run"].items():
 160             trow = ET.SubElement(
 161                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 162             )
 163             tcol = ET.SubElement(
 164                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 165             )
 166             if dut_data.get(u"threads", None) is None:
 167                 tcol.text = u"No Data"
 168                 continue
 169             bold = ET.SubElement(tcol, u"b")
 170             bold.text = dut_name
 171
 172             trow = ET.SubElement(
 173                 tbl, u"tr", attrib=dict(bgcolor=colors[u"body"][0])
 174             )
 175             tcol = ET.SubElement(
 176                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 177             )
 178             bold = ET.SubElement(tcol, u"b")
 179             bold.text = (
 180                 f"Host IP: {dut_data.get(u'host', '')}, "
 181                 f"Socket: {dut_data.get(u'socket', '')}"
 182             )
 183             trow = ET.SubElement(
 184                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 185             )
 186             thead = ET.SubElement(
 187                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 188             )
 189             thead.text = u"\t"
 190
 191             for thread_nr, thread in dut_data[u"threads"].items():
 192                 trow = ET.SubElement(
 193                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 194                 )
 195                 tcol = ET.SubElement(
 196                     trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 197                 )
 198                 bold = ET.SubElement(tcol, u"b")
 199                 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
 200                 trow = ET.SubElement(
 201                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 202                 )
 203                 for idx, col in enumerate(tbl_hdr):
 204                     tcol = ET.SubElement(
 205                         trow, u"td",
 206                         attrib=dict(align=u"right" if idx else u"left")
 207                     )
 208                     font = ET.SubElement(
 209                         tcol, u"font", attrib=dict(size=u"2")
 210                     )
 211                     bold = ET.SubElement(font, u"b")
 212                     bold.text = col
 213                 for row_nr, row in enumerate(thread):
 214                     trow = ET.SubElement(
 215                         tbl, u"tr",
 216                         attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
 217                     )
 218                     for idx, col in enumerate(row):
 219                         tcol = ET.SubElement(
 220                             trow, u"td",
 221                             attrib=dict(align=u"right" if idx else u"left")
 222                         )
 223                         font = ET.SubElement(
 224                             tcol, u"font", attrib=dict(size=u"2")
 225                         )
 226                         if isinstance(col, float):
 227                             font.text = f"{col:.2f}"
 228                         else:
 229                             font.text = str(col)
 230                 trow = ET.SubElement(
 231                     tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 232                 )
 233                 thead = ET.SubElement(
 234                     trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 235                 )
 236                 thead.text = u"\t"
 237
 238         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 239         thead = ET.SubElement(
 240             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 241         )
 242         font = ET.SubElement(
 243             thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 244         )
 245         font.text = u"."
 246
 247         return str(ET.tostring(tbl, encoding=u"unicode"))
 248
 249     for suite in suites.values:
 250         html_table = str()
 251         for test_data in data.values:
 252             if test_data[u"parent"] not in suite[u"name"]:
 253                 continue
 254             html_table += _generate_html_table(test_data)
 255         if not html_table:
 256             continue
 257         try:
 258             file_name = f"{table[u'output-file']}_{suite[u'name']}.rst"
 259             with open(f"{file_name}", u'w') as html_file:
 260                 logging.info(f"    Writing file: {file_name}")
 261                 html_file.write(u".. raw:: html\n\n\t")
 262                 html_file.write(html_table)
 263                 html_file.write(u"\n\t<p><br><br></p>\n")
 264         except KeyError:
 265             logging.warning(u"The output file is not defined.")
 266             return
 267     logging.info(u"  Done.")
 268
 269
 270 def table_details(table, input_data):
 271     """Generate the table(s) with algorithm: table_detailed_test_results
 272     specified in the specification file.
 273
 274     :param table: Table to generate.
 275     :param input_data: Data to process.
 276     :type table: pandas.Series
 277     :type input_data: InputData
 278     """
 279
 280     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 281
 282     # Transform the data
 283     logging.info(
 284         f"    Creating the data set for the {table.get(u'type', u'')} "
 285         f"{table.get(u'title', u'')}."
 286     )
 287     data = input_data.filter_data(table)
 288
 289     # Prepare the header of the tables
 290     header = list()
 291     for column in table[u"columns"]:
 292         header.append(
 293             u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
 294         )
 295
 296     # Generate the data for the table according to the model in the table
 297     # specification
 298     job = list(table[u"data"].keys())[0]
 299     build = str(table[u"data"][job][0])
 300     try:
 301         suites = input_data.suites(job, build)
 302     except KeyError:
 303         logging.error(
 304             u"    No data available. The table will not be generated."
 305         )
 306         return
 307
 308     for suite in suites.values:
 309         # Generate data
 310         suite_name = suite[u"name"]
 311         table_lst = list()
 312         for test in data[job][build].keys():
 313             if data[job][build][test][u"parent"] not in suite_name:
 314                 continue
 315             row_lst = list()
 316             for column in table[u"columns"]:
 317                 try:
 318                     col_data = str(data[job][build][test][column[
 319                         u"data"].split(" ")[1]]).replace(u'"', u'""')
 320                     if column[u"data"].split(u" ")[1] in (u"name", u"msg"):
 321                         col_data = f" |prein| {col_data} |preout| "
 322                     if column[u"data"].split(u" ")[1] in \
 323                         (u"conf-history", u"show-run"):
 324                         col_data = col_data.replace(u" |br| ", u"", 1)
 325                         col_data = f" |prein| {col_data[:-5]} |preout| "
 326                     row_lst.append(f'"{col_data}"')
 327                 except KeyError:
 328                     row_lst.append(u"No data")
 329             table_lst.append(row_lst)
 330
 331         # Write the data to file
 332         if table_lst:
 333             file_name = (
 334                 f"{table[u'output-file']}_{suite_name}"
 335                 f"{table[u'output-file-ext']}"
 336             )
 337             logging.info(f"      Writing file: {file_name}")
 338             with open(file_name, u"wt") as file_handler:
 339                 file_handler.write(u",".join(header) + u"\n")
 340                 for item in table_lst:
 341                     file_handler.write(u",".join(item) + u"\n")
 342
 343     logging.info(u"  Done.")
 344
 345
 346 def table_merged_details(table, input_data):
 347     """Generate the table(s) with algorithm: table_merged_details
 348     specified in the specification file.
 349
 350     :param table: Table to generate.
 351     :param input_data: Data to process.
 352     :type table: pandas.Series
 353     :type input_data: InputData
 354     """
 355
 356     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 357     # Transform the data
 358     logging.info(
 359         f"    Creating the data set for the {table.get(u'type', u'')} "
 360         f"{table.get(u'title', u'')}."
 361     )
 362     data = input_data.filter_data(table, continue_on_error=True)
 363     data = input_data.merge_data(data)
 364     data.sort_index(inplace=True)
 365
 366     logging.info(
 367         f"    Creating the data set for the {table.get(u'type', u'')} "
 368         f"{table.get(u'title', u'')}."
 369     )
 370     suites = input_data.filter_data(
 371         table, continue_on_error=True, data_set=u"suites")
 372     suites = input_data.merge_data(suites)
 373
 374     # Prepare the header of the tables
 375     header = list()
 376     for column in table[u"columns"]:
 377         header.append(
 378             u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
 379         )
 380
 381     for suite in suites.values:
 382         # Generate data
 383         suite_name = suite[u"name"]
 384         table_lst = list()
 385         for test in data.keys():
 386             if data[test][u"parent"] not in suite_name:
 387                 continue
 388             row_lst = list()
 389             for column in table[u"columns"]:
 390                 try:
 391                     col_data = str(data[test][column[
 392                         u"data"].split(u" ")[1]]).replace(u'"', u'""')
 393                     col_data = col_data.replace(
 394                         u"No Data", u"Not Captured     "
 395                     )
 396                     if column[u"data"].split(u" ")[1] in (u"name", u"msg"):
 397                         col_data = f" |prein| {col_data} |preout| "
 398                     if column[u"data"].split(u" ")[1] in \
 399                         (u"conf-history", u"show-run"):
 400                         col_data = col_data.replace(u" |br| ", u"", 1)
 401                         col_data = f" |prein| {col_data[:-5]} |preout| "
 402                     row_lst.append(f'"{col_data}"')
 403                 except KeyError:
 404                     row_lst.append(u'"Not captured"')
 405             table_lst.append(row_lst)
 406
 407         # Write the data to file
 408         if table_lst:
 409             file_name = (
 410                 f"{table[u'output-file']}_{suite_name}"
 411                 f"{table[u'output-file-ext']}"
 412             )
 413             logging.info(f"      Writing file: {file_name}")
 414             with open(file_name, u"wt") as file_handler:
 415                 file_handler.write(u",".join(header) + u"\n")
 416                 for item in table_lst:
 417                     file_handler.write(u",".join(item) + u"\n")
 418
 419     logging.info(u"  Done.")
 420
 421
 422 def _tpc_modify_test_name(test_name):
 423     """Modify a test name by replacing its parts.
 424
 425     :param test_name: Test name to be modified.
 426     :type test_name: str
 427     :returns: Modified test name.
 428     :rtype: str
 429     """
 430     test_name_mod = test_name.\
 431         replace(u"-ndrpdrdisc", u""). \
 432         replace(u"-ndrpdr", u"").\
 433         replace(u"-pdrdisc", u""). \
 434         replace(u"-ndrdisc", u"").\
 435         replace(u"-pdr", u""). \
 436         replace(u"-ndr", u""). \
 437         replace(u"1t1c", u"1c").\
 438         replace(u"2t1c", u"1c"). \
 439         replace(u"2t2c", u"2c").\
 440         replace(u"4t2c", u"2c"). \
 441         replace(u"4t4c", u"4c").\
 442         replace(u"8t4c", u"4c")
 443
 444     return re.sub(REGEX_NIC, u"", test_name_mod)
 445
 446
 447 def _tpc_modify_displayed_test_name(test_name):
 448     """Modify a test name which is displayed in a table by replacing its parts.
 449
 450     :param test_name: Test name to be modified.
 451     :type test_name: str
 452     :returns: Modified test name.
 453     :rtype: str
 454     """
 455     return test_name.\
 456         replace(u"1t1c", u"1c").\
 457         replace(u"2t1c", u"1c"). \
 458         replace(u"2t2c", u"2c").\
 459         replace(u"4t2c", u"2c"). \
 460         replace(u"4t4c", u"4c").\
 461         replace(u"8t4c", u"4c")
 462
 463
 464 def _tpc_insert_data(target, src, include_tests):
 465     """Insert src data to the target structure.
 466
 467     :param target: Target structure where the data is placed.
 468     :param src: Source data to be placed into the target stucture.
 469     :param include_tests: Which results will be included (MRR, NDR, PDR).
 470     :type target: list
 471     :type src: dict
 472     :type include_tests: str
 473     """
 474     try:
 475         if include_tests == u"MRR":
 476             target.append(src[u"result"][u"receive-rate"])
 477         elif include_tests == u"PDR":
 478             target.append(src[u"throughput"][u"PDR"][u"LOWER"])
 479         elif include_tests == u"NDR":
 480             target.append(src[u"throughput"][u"NDR"][u"LOWER"])
 481     except (KeyError, TypeError):
 482         pass
 483
 484
 485 def _tpc_sort_table(table):
 486     """Sort the table this way:
 487
 488     1. Put "New in CSIT-XXXX" at the first place.
 489     2. Put "See footnote" at the second place.
 490     3. Sort the rest by "Delta".
 491
 492     :param table: Table to sort.
 493     :type table: list
 494     :returns: Sorted table.
 495     :rtype: list
 496     """
 497
 498
 499     tbl_new = list()
 500     tbl_see = list()
 501     tbl_delta = list()
 502     for item in table:
 503         if isinstance(item[-1], str):
 504             if u"New in CSIT" in item[-1]:
 505                 tbl_new.append(item)
 506             elif u"See footnote" in item[-1]:
 507                 tbl_see.append(item)
 508         else:
 509             tbl_delta.append(item)
 510
 511     # Sort the tables:
 512     tbl_new.sort(key=lambda rel: rel[0], reverse=False)
 513     tbl_see.sort(key=lambda rel: rel[0], reverse=False)
 514     tbl_see.sort(key=lambda rel: rel[-1], reverse=False)
 515     tbl_delta.sort(key=lambda rel: rel[-1], reverse=True)
 516
 517     # Put the tables together:
 518     table = list()
 519     table.extend(tbl_new)
 520     table.extend(tbl_see)
 521     table.extend(tbl_delta)
 522
 523     return table
 524
 525
 526 def _tpc_generate_html_table(header, data, output_file_name):
 527     """Generate html table from input data with simple sorting possibility.
 528
 529     :param header: Table header.
 530     :param data: Input data to be included in the table. It is a list of lists.
 531         Inner lists are rows in the table. All inner lists must be of the same
 532         length. The length of these lists must be the same as the length of the
 533         header.
 534     :param output_file_name: The name (relative or full path) where the
 535         generated html table is written.
 536     :type header: list
 537     :type data: list of lists
 538     :type output_file_name: str
 539     """
 540
 541     df_data = pd.DataFrame(data, columns=header)
 542
 543     df_sorted = [df_data.sort_values(
 544         by=[key, header[0]], ascending=[True, True]
 545         if key != header[0] else [False, True]) for key in header]
 546     df_sorted_rev = [df_data.sort_values(
 547         by=[key, header[0]], ascending=[False, True]
 548         if key != header[0] else [True, True]) for key in header]
 549     df_sorted.extend(df_sorted_rev)
 550
 551     fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
 552                    for idx in range(len(df_data))]]
 553     table_header = dict(
 554         values=[f"<b>{item}</b>" for item in header],
 555         fill_color=u"#7eade7",
 556         align=[u"left", u"center"]
 557     )
 558
 559     fig = go.Figure()
 560
 561     for table in df_sorted:
 562         columns = [table.get(col) for col in header]
 563         fig.add_trace(
 564             go.Table(
 565                 columnwidth=[30, 10],
 566                 header=table_header,
 567                 cells=dict(
 568                     values=columns,
 569                     fill_color=fill_color,
 570                     align=[u"left", u"right"]
 571                 )
 572             )
 573         )
 574
 575     buttons = list()
 576     menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
 577     menu_items_rev = [f"<b>{itm}</b> (descending)" for itm in header]
 578     menu_items.extend(menu_items_rev)
 579     for idx, hdr in enumerate(menu_items):
 580         visible = [False, ] * len(menu_items)
 581         visible[idx] = True
 582         buttons.append(
 583             dict(
 584                 label=hdr.replace(u" [Mpps]", u""),
 585                 method=u"update",
 586                 args=[{u"visible": visible}],
 587             )
 588         )
 589
 590     fig.update_layout(
 591         updatemenus=[
 592             go.layout.Updatemenu(
 593                 type=u"dropdown",
 594                 direction=u"down",
 595                 x=0.03,
 596                 xanchor=u"left",
 597                 y=1.045,
 598                 yanchor=u"top",
 599                 active=len(menu_items) - 1,
 600                 buttons=list(buttons)
 601             )
 602         ],
 603         annotations=[
 604             go.layout.Annotation(
 605                 text=u"<b>Sort by:</b>",
 606                 x=0,
 607                 xref=u"paper",
 608                 y=1.035,
 609                 yref=u"paper",
 610                 align=u"left",
 611                 showarrow=False
 612             )
 613         ]
 614     )
 615
 616     ploff.plot(fig, show_link=False, auto_open=False, filename=output_file_name)
 617
 618
 619 def table_perf_comparison(table, input_data):
 620     """Generate the table(s) with algorithm: table_perf_comparison
 621     specified in the specification file.
 622
 623     :param table: Table to generate.
 624     :param input_data: Data to process.
 625     :type table: pandas.Series
 626     :type input_data: InputData
 627     """
 628
 629     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 630
 631     # Transform the data
 632     logging.info(
 633         f"    Creating the data set for the {table.get(u'type', u'')} "
 634         f"{table.get(u'title', u'')}."
 635     )
 636     data = input_data.filter_data(table, continue_on_error=True)
 637
 638     # Prepare the header of the tables
 639     try:
 640         header = [u"Test case", ]
 641
 642         if table[u"include-tests"] == u"MRR":
 643             hdr_param = u"Rec Rate"
 644         else:
 645             hdr_param = u"Thput"
 646
 647         history = table.get(u"history", list())
 648         for item in history:
 649             header.extend(
 650                 [
 651                     f"{item[u'title']} {hdr_param} [Mpps]",
 652                     f"{item[u'title']} Stdev [Mpps]"
 653                 ]
 654             )
 655         header.extend(
 656             [
 657                 f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
 658                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
 659                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
 660                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
 661                 u"Delta [%]"
 662             ]
 663         )
 664         header_str = u",".join(header) + u"\n"
 665     except (AttributeError, KeyError) as err:
 666         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 667         return
 668
 669     # Prepare data to the table:
 670     tbl_dict = dict()
 671     # topo = ""
 672     for job, builds in table[u"reference"][u"data"].items():
 673         # topo = u"2n-skx" if u"2n-skx" in job else u""
 674         for build in builds:
 675             for tst_name, tst_data in data[job][str(build)].items():
 676                 tst_name_mod = _tpc_modify_test_name(tst_name)
 677                 if u"across topologies" in table[u"title"].lower():
 678                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 679                 if tbl_dict.get(tst_name_mod, None) is None:
 680                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
 681                     nic = groups.group(0) if groups else u""
 682                     name = \
 683                         f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 684                     if u"across testbeds" in table[u"title"].lower() or \
 685                             u"across topologies" in table[u"title"].lower():
 686                         name = _tpc_modify_displayed_test_name(name)
 687                     tbl_dict[tst_name_mod] = {
 688                         u"name": name,
 689                         u"ref-data": list(),
 690                         u"cmp-data": list()
 691                     }
 692                 _tpc_insert_data(target=tbl_dict[tst_name_mod][u"ref-data"],
 693                                  src=tst_data,
 694                                  include_tests=table[u"include-tests"])
 695
 696     replacement = table[u"reference"].get(u"data-replacement", None)
 697     if replacement:
 698         create_new_list = True
 699         rpl_data = input_data.filter_data(
 700             table, data=replacement, continue_on_error=True)
 701         for job, builds in replacement.items():
 702             for build in builds:
 703                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 704                     tst_name_mod = _tpc_modify_test_name(tst_name)
 705                     if u"across topologies" in table[u"title"].lower():
 706                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 707                     if tbl_dict.get(tst_name_mod, None) is None:
 708                         name = \
 709                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 710                         if u"across testbeds" in table[u"title"].lower() or \
 711                                 u"across topologies" in table[u"title"].lower():
 712                             name = _tpc_modify_displayed_test_name(name)
 713                         tbl_dict[tst_name_mod] = {
 714                             u"name": name,
 715                             u"ref-data": list(),
 716                             u"cmp-data": list()
 717                         }
 718                     if create_new_list:
 719                         create_new_list = False
 720                         tbl_dict[tst_name_mod][u"ref-data"] = list()
 721
 722                     _tpc_insert_data(
 723                         target=tbl_dict[tst_name_mod][u"ref-data"],
 724                         src=tst_data,
 725                         include_tests=table[u"include-tests"]
 726                     )
 727
 728     for job, builds in table[u"compare"][u"data"].items():
 729         for build in builds:
 730             for tst_name, tst_data in data[job][str(build)].items():
 731                 tst_name_mod = _tpc_modify_test_name(tst_name)
 732                 if u"across topologies" in table[u"title"].lower():
 733                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 734                 if tbl_dict.get(tst_name_mod, None) is None:
 735                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
 736                     nic = groups.group(0) if groups else u""
 737                     name = \
 738                         f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 739                     if u"across testbeds" in table[u"title"].lower() or \
 740                             u"across topologies" in table[u"title"].lower():
 741                         name = _tpc_modify_displayed_test_name(name)
 742                     tbl_dict[tst_name_mod] = {
 743                         u"name": name,
 744                         u"ref-data": list(),
 745                         u"cmp-data": list()
 746                     }
 747                 _tpc_insert_data(
 748                     target=tbl_dict[tst_name_mod][u"cmp-data"],
 749                     src=tst_data,
 750                     include_tests=table[u"include-tests"]
 751                 )
 752
 753     replacement = table[u"compare"].get(u"data-replacement", None)
 754     if replacement:
 755         create_new_list = True
 756         rpl_data = input_data.filter_data(
 757             table, data=replacement, continue_on_error=True)
 758         for job, builds in replacement.items():
 759             for build in builds:
 760                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 761                     tst_name_mod = _tpc_modify_test_name(tst_name)
 762                     if u"across topologies" in table[u"title"].lower():
 763                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 764                     if tbl_dict.get(tst_name_mod, None) is None:
 765                         name = \
 766                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 767                         if u"across testbeds" in table[u"title"].lower() or \
 768                                 u"across topologies" in table[u"title"].lower():
 769                             name = _tpc_modify_displayed_test_name(name)
 770                         tbl_dict[tst_name_mod] = {
 771                             u"name": name,
 772                             u"ref-data": list(),
 773                             u"cmp-data": list()
 774                         }
 775                     if create_new_list:
 776                         create_new_list = False
 777                         tbl_dict[tst_name_mod][u"cmp-data"] = list()
 778
 779                     _tpc_insert_data(
 780                         target=tbl_dict[tst_name_mod][u"cmp-data"],
 781                         src=tst_data,
 782                         include_tests=table[u"include-tests"]
 783                     )
 784
 785     for item in history:
 786         for job, builds in item[u"data"].items():
 787             for build in builds:
 788                 for tst_name, tst_data in data[job][str(build)].items():
 789                     tst_name_mod = _tpc_modify_test_name(tst_name)
 790                     if u"across topologies" in table[u"title"].lower():
 791                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 792                     if tbl_dict.get(tst_name_mod, None) is None:
 793                         continue
 794                     if tbl_dict[tst_name_mod].get(u"history", None) is None:
 795                         tbl_dict[tst_name_mod][u"history"] = OrderedDict()
 796                     if tbl_dict[tst_name_mod][u"history"].\
 797                             get(item[u"title"], None) is None:
 798                         tbl_dict[tst_name_mod][u"history"][item[
 799                             u"title"]] = list()
 800                     try:
 801                         if table[u"include-tests"] == u"MRR":
 802                             res = tst_data[u"result"][u"receive-rate"]
 803                         elif table[u"include-tests"] == u"PDR":
 804                             res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
 805                         elif table[u"include-tests"] == u"NDR":
 806                             res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
 807                         else:
 808                             continue
 809                         tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
 810                             append(res)
 811                     except (TypeError, KeyError):
 812                         pass
 813
 814     tbl_lst = list()
 815     footnote = False
 816     for tst_name in tbl_dict:
 817         item = [tbl_dict[tst_name][u"name"], ]
 818         if history:
 819             if tbl_dict[tst_name].get(u"history", None) is not None:
 820                 for hist_data in tbl_dict[tst_name][u"history"].values():
 821                     if hist_data:
 822                         item.append(round(mean(hist_data) / 1000000, 2))
 823                         item.append(round(stdev(hist_data) / 1000000, 2))
 824                     else:
 825                         item.extend([u"Not tested", u"Not tested"])
 826             else:
 827                 item.extend([u"Not tested", u"Not tested"])
 828         data_t = tbl_dict[tst_name][u"ref-data"]
 829         if data_t:
 830             item.append(round(mean(data_t) / 1000000, 2))
 831             item.append(round(stdev(data_t) / 1000000, 2))
 832         else:
 833             item.extend([u"Not tested", u"Not tested"])
 834         data_t = tbl_dict[tst_name][u"cmp-data"]
 835         if data_t:
 836             item.append(round(mean(data_t) / 1000000, 2))
 837             item.append(round(stdev(data_t) / 1000000, 2))
 838         else:
 839             item.extend([u"Not tested", u"Not tested"])
 840         if item[-2] == u"Not tested":
 841             pass
 842         elif item[-4] == u"Not tested":
 843             item.append(u"New in CSIT-2001")
 844         # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
 845         #     item.append(u"See footnote [1]")
 846         #     footnote = True
 847         elif item[-4] != 0:
 848             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 849         if (len(item) == len(header)) and (item[-3] != u"Not tested"):
 850             tbl_lst.append(item)
 851
 852     tbl_lst = _tpc_sort_table(tbl_lst)
 853
 854     # Generate csv tables:
 855     csv_file = f"{table[u'output-file']}.csv"
 856     with open(csv_file, u"wt") as file_handler:
 857         file_handler.write(header_str)
 858         for test in tbl_lst:
 859             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
 860
 861     txt_file_name = f"{table[u'output-file']}.txt"
 862     convert_csv_to_pretty_txt(csv_file, txt_file_name)
 863
 864     if footnote:
 865         with open(txt_file_name, u'a') as txt_file:
 866             txt_file.writelines([
 867                 u"\nFootnotes:\n",
 868                 u"[1] CSIT-1908 changed test methodology of dot1q tests in "
 869                 u"2-node testbeds, dot1q encapsulation is now used on both "
 870                 u"links of SUT.\n",
 871                 u"    Previously dot1q was used only on a single link with the "
 872                 u"other link carrying untagged Ethernet frames. This changes "
 873                 u"results\n",
 874                 u"    in slightly lower throughput in CSIT-1908 for these "
 875                 u"tests. See release notes."
 876             ])
 877
 878     # Generate html table:
 879     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
 880
 881
 882 def table_perf_comparison_nic(table, input_data):
 883     """Generate the table(s) with algorithm: table_perf_comparison
 884     specified in the specification file.
 885
 886     :param table: Table to generate.
 887     :param input_data: Data to process.
 888     :type table: pandas.Series
 889     :type input_data: InputData
 890     """
 891
 892     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 893
 894     # Transform the data
 895     logging.info(
 896         f"    Creating the data set for the {table.get(u'type', u'')} "
 897         f"{table.get(u'title', u'')}."
 898     )
 899     data = input_data.filter_data(table, continue_on_error=True)
 900
 901     # Prepare the header of the tables
 902     try:
 903         header = [u"Test case", ]
 904
 905         if table[u"include-tests"] == u"MRR":
 906             hdr_param = u"Rec Rate"
 907         else:
 908             hdr_param = u"Thput"
 909
 910         history = table.get(u"history", list())
 911         for item in history:
 912             header.extend(
 913                 [
 914                     f"{item[u'title']} {hdr_param} [Mpps]",
 915                     f"{item[u'title']} Stdev [Mpps]"
 916                 ]
 917             )
 918         header.extend(
 919             [
 920                 f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
 921                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
 922                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
 923                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
 924                 u"Delta [%]"
 925             ]
 926         )
 927         header_str = u",".join(header) + u"\n"
 928     except (AttributeError, KeyError) as err:
 929         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 930         return
 931
 932     # Prepare data to the table:
 933     tbl_dict = dict()
 934     # topo = u""
 935     for job, builds in table[u"reference"][u"data"].items():
 936         # topo = u"2n-skx" if u"2n-skx" in job else u""
 937         for build in builds:
 938             for tst_name, tst_data in data[job][str(build)].items():
 939                 if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
 940                     continue
 941                 tst_name_mod = _tpc_modify_test_name(tst_name)
 942                 if u"across topologies" in table[u"title"].lower():
 943                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 944                 if tbl_dict.get(tst_name_mod, None) is None:
 945                     name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 946                     if u"across testbeds" in table[u"title"].lower() or \
 947                             u"across topologies" in table[u"title"].lower():
 948                         name = _tpc_modify_displayed_test_name(name)
 949                     tbl_dict[tst_name_mod] = {
 950                         u"name": name,
 951                         u"ref-data": list(),
 952                         u"cmp-data": list()
 953                     }
 954                 _tpc_insert_data(
 955                     target=tbl_dict[tst_name_mod][u"ref-data"],
 956                     src=tst_data,
 957                     include_tests=table[u"include-tests"]
 958                 )
 959
 960     replacement = table[u"reference"].get(u"data-replacement", None)
 961     if replacement:
 962         create_new_list = True
 963         rpl_data = input_data.filter_data(
 964             table, data=replacement, continue_on_error=True)
 965         for job, builds in replacement.items():
 966             for build in builds:
 967                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 968                     if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
 969                         continue
 970                     tst_name_mod = _tpc_modify_test_name(tst_name)
 971                     if u"across topologies" in table[u"title"].lower():
 972                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 973                     if tbl_dict.get(tst_name_mod, None) is None:
 974                         name = \
 975                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 976                         if u"across testbeds" in table[u"title"].lower() or \
 977                                 u"across topologies" in table[u"title"].lower():
 978                             name = _tpc_modify_displayed_test_name(name)
 979                         tbl_dict[tst_name_mod] = {
 980                             u"name": name,
 981                             u"ref-data": list(),
 982                             u"cmp-data": list()
 983                         }
 984                     if create_new_list:
 985                         create_new_list = False
 986                         tbl_dict[tst_name_mod][u"ref-data"] = list()
 987
 988                     _tpc_insert_data(
 989                         target=tbl_dict[tst_name_mod][u"ref-data"],
 990                         src=tst_data,
 991                         include_tests=table[u"include-tests"]
 992                     )
 993
 994     for job, builds in table[u"compare"][u"data"].items():
 995         for build in builds:
 996             for tst_name, tst_data in data[job][str(build)].items():
 997                 if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
 998                     continue
 999                 tst_name_mod = _tpc_modify_test_name(tst_name)
1000                 if u"across topologies" in table[u"title"].lower():
1001                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1002                 if tbl_dict.get(tst_name_mod, None) is None:
1003                     name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
1004                     if u"across testbeds" in table[u"title"].lower() or \
1005                             u"across topologies" in table[u"title"].lower():
1006                         name = _tpc_modify_displayed_test_name(name)
1007                     tbl_dict[tst_name_mod] = {
1008                         u"name": name,
1009                         u"ref-data": list(),
1010                         u"cmp-data": list()
1011                     }
1012                 _tpc_insert_data(
1013                     target=tbl_dict[tst_name_mod][u"cmp-data"],
1014                     src=tst_data,
1015                     include_tests=table[u"include-tests"]
1016                 )
1017
1018     replacement = table[u"compare"].get(u"data-replacement", None)
1019     if replacement:
1020         create_new_list = True
1021         rpl_data = input_data.filter_data(
1022             table, data=replacement, continue_on_error=True)
1023         for job, builds in replacement.items():
1024             for build in builds:
1025                 for tst_name, tst_data in rpl_data[job][str(build)].items():
1026                     if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
1027                         continue
1028                     tst_name_mod = _tpc_modify_test_name(tst_name)
1029                     if u"across topologies" in table[u"title"].lower():
1030                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1031                     if tbl_dict.get(tst_name_mod, None) is None:
1032                         name = \
1033                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
1034                         if u"across testbeds" in table[u"title"].lower() or \
1035                                 u"across topologies" in table[u"title"].lower():
1036                             name = _tpc_modify_displayed_test_name(name)
1037                         tbl_dict[tst_name_mod] = {
1038                             u"name": name,
1039                             u"ref-data": list(),
1040                             u"cmp-data": list()
1041                         }
1042                     if create_new_list:
1043                         create_new_list = False
1044                         tbl_dict[tst_name_mod][u"cmp-data"] = list()
1045
1046                     _tpc_insert_data(
1047                         target=tbl_dict[tst_name_mod][u"cmp-data"],
1048                         src=tst_data,
1049                         include_tests=table[u"include-tests"]
1050                     )
1051
1052     for item in history:
1053         for job, builds in item[u"data"].items():
1054             for build in builds:
1055                 for tst_name, tst_data in data[job][str(build)].items():
1056                     if item[u"nic"] not in tst_data[u"tags"]:
1057                         continue
1058                     tst_name_mod = _tpc_modify_test_name(tst_name)
1059                     if u"across topologies" in table[u"title"].lower():
1060                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1061                     if tbl_dict.get(tst_name_mod, None) is None:
1062                         continue
1063                     if tbl_dict[tst_name_mod].get(u"history", None) is None:
1064                         tbl_dict[tst_name_mod][u"history"] = OrderedDict()
1065                     if tbl_dict[tst_name_mod][u"history"].\
1066                             get(item[u"title"], None) is None:
1067                         tbl_dict[tst_name_mod][u"history"][item[
1068                             u"title"]] = list()
1069                     try:
1070                         if table[u"include-tests"] == u"MRR":
1071                             res = tst_data[u"result"][u"receive-rate"]
1072                         elif table[u"include-tests"] == u"PDR":
1073                             res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
1074                         elif table[u"include-tests"] == u"NDR":
1075                             res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
1076                         else:
1077                             continue
1078                         tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
1079                             append(res)
1080                     except (TypeError, KeyError):
1081                         pass
1082
1083     tbl_lst = list()
1084     footnote = False
1085     for tst_name in tbl_dict:
1086         item = [tbl_dict[tst_name][u"name"], ]
1087         if history:
1088             if tbl_dict[tst_name].get(u"history", None) is not None:
1089                 for hist_data in tbl_dict[tst_name][u"history"].values():
1090                     if hist_data:
1091                         item.append(round(mean(hist_data) / 1000000, 2))
1092                         item.append(round(stdev(hist_data) / 1000000, 2))
1093                     else:
1094                         item.extend([u"Not tested", u"Not tested"])
1095             else:
1096                 item.extend([u"Not tested", u"Not tested"])
1097         data_t = tbl_dict[tst_name][u"ref-data"]
1098         if data_t:
1099             item.append(round(mean(data_t) / 1000000, 2))
1100             item.append(round(stdev(data_t) / 1000000, 2))
1101         else:
1102             item.extend([u"Not tested", u"Not tested"])
1103         data_t = tbl_dict[tst_name][u"cmp-data"]
1104         if data_t:
1105             item.append(round(mean(data_t) / 1000000, 2))
1106             item.append(round(stdev(data_t) / 1000000, 2))
1107         else:
1108             item.extend([u"Not tested", u"Not tested"])
1109         if item[-2] == u"Not tested":
1110             pass
1111         elif item[-4] == u"Not tested":
1112             item.append(u"New in CSIT-2001")
1113         # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
1114         #     item.append(u"See footnote [1]")
1115         #     footnote = True
1116         elif item[-4] != 0:
1117             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
1118         if (len(item) == len(header)) and (item[-3] != u"Not tested"):
1119             tbl_lst.append(item)
1120
1121     tbl_lst = _tpc_sort_table(tbl_lst)
1122
1123     # Generate csv tables:
1124     csv_file = f"{table[u'output-file']}.csv"
1125     with open(csv_file, u"wt") as file_handler:
1126         file_handler.write(header_str)
1127         for test in tbl_lst:
1128             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
1129
1130     txt_file_name = f"{table[u'output-file']}.txt"
1131     convert_csv_to_pretty_txt(csv_file, txt_file_name)
1132
1133     if footnote:
1134         with open(txt_file_name, u'a') as txt_file:
1135             txt_file.writelines([
1136                 u"\nFootnotes:\n",
1137                 u"[1] CSIT-1908 changed test methodology of dot1q tests in "
1138                 u"2-node testbeds, dot1q encapsulation is now used on both "
1139                 u"links of SUT.\n",
1140                 u"    Previously dot1q was used only on a single link with the "
1141                 u"other link carrying untagged Ethernet frames. This changes "
1142                 u"results\n",
1143                 u"    in slightly lower throughput in CSIT-1908 for these "
1144                 u"tests. See release notes."
1145             ])
1146
1147     # Generate html table:
1148     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
1149
1150
1151 def table_nics_comparison(table, input_data):
1152     """Generate the table(s) with algorithm: table_nics_comparison
1153     specified in the specification file.
1154
1155     :param table: Table to generate.
1156     :param input_data: Data to process.
1157     :type table: pandas.Series
1158     :type input_data: InputData
1159     """
1160
1161     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1162
1163     # Transform the data
1164     logging.info(
1165         f"    Creating the data set for the {table.get(u'type', u'')} "
1166         f"{table.get(u'title', u'')}."
1167     )
1168     data = input_data.filter_data(table, continue_on_error=True)
1169
1170     # Prepare the header of the tables
1171     try:
1172         header = [u"Test case", ]
1173
1174         if table[u"include-tests"] == u"MRR":
1175             hdr_param = u"Rec Rate"
1176         else:
1177             hdr_param = u"Thput"
1178
1179         header.extend(
1180             [
1181                 f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
1182                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
1183                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
1184                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
1185                 u"Delta [%]"
1186             ]
1187         )
1188
1189     except (AttributeError, KeyError) as err:
1190         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1191         return
1192
1193     # Prepare data to the table:
1194     tbl_dict = dict()
1195     for job, builds in table[u"data"].items():
1196         for build in builds:
1197             for tst_name, tst_data in data[job][str(build)].items():
1198                 tst_name_mod = _tpc_modify_test_name(tst_name)
1199                 if tbl_dict.get(tst_name_mod, None) is None:
1200                     name = u"-".join(tst_data[u"name"].split(u"-")[:-1])
1201                     tbl_dict[tst_name_mod] = {
1202                         u"name": name,
1203                         u"ref-data": list(),
1204                         u"cmp-data": list()
1205                     }
1206                 try:
1207                     result = None
1208                     if table[u"include-tests"] == u"MRR":
1209                         result = tst_data[u"result"][u"receive-rate"]
1210                     elif table[u"include-tests"] == u"PDR":
1211                         result = tst_data[u"throughput"][u"PDR"][u"LOWER"]
1212                     elif table[u"include-tests"] == u"NDR":
1213                         result = tst_data[u"throughput"][u"NDR"][u"LOWER"]
1214                     else:
1215                         continue
1216
1217                     if result and \
1218                             table[u"reference"][u"nic"] in tst_data[u"tags"]:
1219                         tbl_dict[tst_name_mod][u"ref-data"].append(result)
1220                     elif result and \
1221                             table[u"compare"][u"nic"] in tst_data[u"tags"]:
1222                         tbl_dict[tst_name_mod][u"cmp-data"].append(result)
1223                 except (TypeError, KeyError) as err:
1224                     logging.debug(f"No data for {tst_name}\n{repr(err)}")
1225                     # No data in output.xml for this test
1226
1227     tbl_lst = list()
1228     for tst_name in tbl_dict:
1229         item = [tbl_dict[tst_name][u"name"], ]
1230         data_t = tbl_dict[tst_name][u"ref-data"]
1231         if data_t:
1232             item.append(round(mean(data_t) / 1000000, 2))
1233             item.append(round(stdev(data_t) / 1000000, 2))
1234         else:
1235             item.extend([None, None])
1236         data_t = tbl_dict[tst_name][u"cmp-data"]
1237         if data_t:
1238             item.append(round(mean(data_t) / 1000000, 2))
1239             item.append(round(stdev(data_t) / 1000000, 2))
1240         else:
1241             item.extend([None, None])
1242         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
1243             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
1244         if len(item) == len(header):
1245             tbl_lst.append(item)
1246
1247     # Sort the table according to the relative change
1248     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1249
1250     # Generate csv tables:
1251     with open(f"{table[u'output-file']}.csv", u"wt") as file_handler:
1252         file_handler.write(u",".join(header) + u"\n")
1253         for test in tbl_lst:
1254             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
1255
1256     convert_csv_to_pretty_txt(f"{table[u'output-file']}.csv",
1257                               f"{table[u'output-file']}.txt")
1258
1259     # Generate html table:
1260     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
1261
1262
1263 def table_soak_vs_ndr(table, input_data):
1264     """Generate the table(s) with algorithm: table_soak_vs_ndr
1265     specified in the specification file.
1266
1267     :param table: Table to generate.
1268     :param input_data: Data to process.
1269     :type table: pandas.Series
1270     :type input_data: InputData
1271     """
1272
1273     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1274
1275     # Transform the data
1276     logging.info(
1277         f"    Creating the data set for the {table.get(u'type', u'')} "
1278         f"{table.get(u'title', u'')}."
1279     )
1280     data = input_data.filter_data(table, continue_on_error=True)
1281
1282     # Prepare the header of the table
1283     try:
1284         header = [
1285             u"Test case",
1286             f"{table[u'reference'][u'title']} Thput [Mpps]",
1287             f"{table[u'reference'][u'title']} Stdev [Mpps]",
1288             f"{table[u'compare'][u'title']} Thput [Mpps]",
1289             f"{table[u'compare'][u'title']} Stdev [Mpps]",
1290             u"Delta [%]", u"Stdev of delta [%]"
1291         ]
1292         header_str = u",".join(header) + u"\n"
1293     except (AttributeError, KeyError) as err:
1294         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1295         return
1296
1297     # Create a list of available SOAK test results:
1298     tbl_dict = dict()
1299     for job, builds in table[u"compare"][u"data"].items():
1300         for build in builds:
1301             for tst_name, tst_data in data[job][str(build)].items():
1302                 if tst_data[u"type"] == u"SOAK":
1303                     tst_name_mod = tst_name.replace(u"-soak", u"")
1304                     if tbl_dict.get(tst_name_mod, None) is None:
1305                         groups = re.search(REGEX_NIC, tst_data[u"parent"])
1306                         nic = groups.group(0) if groups else u""
1307                         name = (
1308                             f"{nic}-"
1309                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
1310                         )
1311                         tbl_dict[tst_name_mod] = {
1312                             u"name": name,
1313                             u"ref-data": list(),
1314                             u"cmp-data": list()
1315                         }
1316                     try:
1317                         tbl_dict[tst_name_mod][u"cmp-data"].append(
1318                             tst_data[u"throughput"][u"LOWER"])
1319                     except (KeyError, TypeError):
1320                         pass
1321     tests_lst = tbl_dict.keys()
1322
1323     # Add corresponding NDR test results:
1324     for job, builds in table[u"reference"][u"data"].items():
1325         for build in builds:
1326             for tst_name, tst_data in data[job][str(build)].items():
1327                 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
1328                     replace(u"-mrr", u"")
1329                 if tst_name_mod not in tests_lst:
1330                     continue
1331                 try:
1332                     if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
1333                         continue
1334                     if table[u"include-tests"] == u"MRR":
1335                         result = tst_data[u"result"][u"receive-rate"]
1336                     elif table[u"include-tests"] == u"PDR":
1337                         result = \
1338                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
1339                     elif table[u"include-tests"] == u"NDR":
1340                         result = \
1341                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
1342                     else:
1343                         result = None
1344                     if result is not None:
1345                         tbl_dict[tst_name_mod][u"ref-data"].append(
1346                             result)
1347                 except (KeyError, TypeError):
1348                     continue
1349
1350     tbl_lst = list()
1351     for tst_name in tbl_dict:
1352         item = [tbl_dict[tst_name][u"name"], ]
1353         data_r = tbl_dict[tst_name][u"ref-data"]
1354         if data_r:
1355             data_r_mean = mean(data_r)
1356             item.append(round(data_r_mean / 1000000, 2))
1357             data_r_stdev = stdev(data_r)
1358             item.append(round(data_r_stdev / 1000000, 2))
1359         else:
1360             data_r_mean = None
1361             data_r_stdev = None
1362             item.extend([None, None])
1363         data_c = tbl_dict[tst_name][u"cmp-data"]
1364         if data_c:
1365             data_c_mean = mean(data_c)
1366             item.append(round(data_c_mean / 1000000, 2))
1367             data_c_stdev = stdev(data_c)
1368             item.append(round(data_c_stdev / 1000000, 2))
1369         else:
1370             data_c_mean = None
1371             data_c_stdev = None
1372             item.extend([None, None])
1373         if data_r_mean and data_c_mean:
1374             delta, d_stdev = relative_change_stdev(
1375                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
1376             item.append(round(delta, 2))
1377             item.append(round(d_stdev, 2))
1378             tbl_lst.append(item)
1379
1380     # Sort the table according to the relative change
1381     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1382
1383     # Generate csv tables:
1384     csv_file = f"{table[u'output-file']}.csv"
1385     with open(csv_file, u"wt") as file_handler:
1386         file_handler.write(header_str)
1387         for test in tbl_lst:
1388             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
1389
1390     convert_csv_to_pretty_txt(csv_file, f"{table[u'output-file']}.txt")
1391
1392     # Generate html table:
1393     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
1394
1395
1396 def table_perf_trending_dash(table, input_data):
1397     """Generate the table(s) with algorithm:
1398     table_perf_trending_dash
1399     specified in the specification file.
1400
1401     :param table: Table to generate.
1402     :param input_data: Data to process.
1403     :type table: pandas.Series
1404     :type input_data: InputData
1405     """
1406
1407     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1408
1409     # Transform the data
1410     logging.info(
1411         f"    Creating the data set for the {table.get(u'type', u'')} "
1412         f"{table.get(u'title', u'')}."
1413     )
1414     data = input_data.filter_data(table, continue_on_error=True)
1415
1416     # Prepare the header of the tables
1417     header = [
1418         u"Test Case",
1419         u"Trend [Mpps]",
1420         u"Short-Term Change [%]",
1421         u"Long-Term Change [%]",
1422         u"Regressions [#]",
1423         u"Progressions [#]"
1424     ]
1425     header_str = u",".join(header) + u"\n"
1426
1427     # Prepare data to the table:
1428     tbl_dict = dict()
1429     for job, builds in table[u"data"].items():
1430         for build in builds:
1431             for tst_name, tst_data in data[job][str(build)].items():
1432                 if tst_name.lower() in table.get(u"ignore-list", list()):
1433                     continue
1434                 if tbl_dict.get(tst_name, None) is None:
1435                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1436                     if not groups:
1437                         continue
1438                     nic = groups.group(0)
1439                     tbl_dict[tst_name] = {
1440                         u"name": f"{nic}-{tst_data[u'name']}",
1441                         u"data": OrderedDict()
1442                     }
1443                 try:
1444                     tbl_dict[tst_name][u"data"][str(build)] = \
1445                         tst_data[u"result"][u"receive-rate"]
1446                 except (TypeError, KeyError):
1447                     pass  # No data in output.xml for this test
1448
1449     tbl_lst = list()
1450     for tst_name in tbl_dict:
1451         data_t = tbl_dict[tst_name][u"data"]
1452         if len(data_t) < 2:
1453             continue
1454
1455         classification_lst, avgs = classify_anomalies(data_t)
1456
1457         win_size = min(len(data_t), table[u"window"])
1458         long_win_size = min(len(data_t), table[u"long-trend-window"])
1459
1460         try:
1461             max_long_avg = max(
1462                 [x for x in avgs[-long_win_size:-win_size]
1463                  if not isnan(x)])
1464         except ValueError:
1465             max_long_avg = nan
1466         last_avg = avgs[-1]
1467         avg_week_ago = avgs[max(-win_size, -len(avgs))]
1468
1469         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
1470             rel_change_last = nan
1471         else:
1472             rel_change_last = round(
1473                 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
1474
1475         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
1476             rel_change_long = nan
1477         else:
1478             rel_change_long = round(
1479                 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
1480
1481         if classification_lst:
1482             if isnan(rel_change_last) and isnan(rel_change_long):
1483                 continue
1484             if isnan(last_avg) or isnan(rel_change_last) or \
1485                     isnan(rel_change_long):
1486                 continue
1487             tbl_lst.append(
1488                 [tbl_dict[tst_name][u"name"],
1489                  round(last_avg / 1000000, 2),
1490                  rel_change_last,
1491                  rel_change_long,
1492                  classification_lst[-win_size:].count(u"regression"),
1493                  classification_lst[-win_size:].count(u"progression")])
1494
1495     tbl_lst.sort(key=lambda rel: rel[0])
1496
1497     tbl_sorted = list()
1498     for nrr in range(table[u"window"], -1, -1):
1499         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
1500         for nrp in range(table[u"window"], -1, -1):
1501             tbl_out = [item for item in tbl_reg if item[5] == nrp]
1502             tbl_out.sort(key=lambda rel: rel[2])
1503             tbl_sorted.extend(tbl_out)
1504
1505     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1506
1507     logging.info(f"    Writing file: {file_name}")
1508     with open(file_name, u"wt") as file_handler:
1509         file_handler.write(header_str)
1510         for test in tbl_sorted:
1511             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1512
1513     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1514     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1515
1516
1517 def _generate_url(testbed, test_name):
1518     """Generate URL to a trending plot from the name of the test case.
1519
1520     :param testbed: The testbed used for testing.
1521     :param test_name: The name of the test case.
1522     :type testbed: str
1523     :type test_name: str
1524     :returns: The URL to the plot with the trending data for the given test
1525         case.
1526     :rtype str
1527     """
1528
1529     if u"x520" in test_name:
1530         nic = u"x520"
1531     elif u"x710" in test_name:
1532         nic = u"x710"
1533     elif u"xl710" in test_name:
1534         nic = u"xl710"
1535     elif u"xxv710" in test_name:
1536         nic = u"xxv710"
1537     elif u"vic1227" in test_name:
1538         nic = u"vic1227"
1539     elif u"vic1385" in test_name:
1540         nic = u"vic1385"
1541     elif u"x553" in test_name:
1542         nic = u"x553"
1543     elif u"cx556" in test_name or u"cx556a" in test_name:
1544         nic = u"cx556a"
1545     else:
1546         nic = u""
1547
1548     if u"64b" in test_name:
1549         frame_size = u"64b"
1550     elif u"78b" in test_name:
1551         frame_size = u"78b"
1552     elif u"imix" in test_name:
1553         frame_size = u"imix"
1554     elif u"9000b" in test_name:
1555         frame_size = u"9000b"
1556     elif u"1518b" in test_name:
1557         frame_size = u"1518b"
1558     elif u"114b" in test_name:
1559         frame_size = u"114b"
1560     else:
1561         frame_size = u""
1562
1563     if u"1t1c" in test_name or \
1564         (u"-1c-" in test_name and
1565          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1566         cores = u"1t1c"
1567     elif u"2t2c" in test_name or \
1568          (u"-2c-" in test_name and
1569           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1570         cores = u"2t2c"
1571     elif u"4t4c" in test_name or \
1572          (u"-4c-" in test_name and
1573           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1574         cores = u"4t4c"
1575     elif u"2t1c" in test_name or \
1576          (u"-1c-" in test_name and
1577           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1578         cores = u"2t1c"
1579     elif u"4t2c" in test_name or \
1580          (u"-2c-" in test_name and
1581           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1582         cores = u"4t2c"
1583     elif u"8t4c" in test_name or \
1584          (u"-4c-" in test_name and
1585           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1586         cores = u"8t4c"
1587     else:
1588         cores = u""
1589
1590     if u"testpmd" in test_name:
1591         driver = u"testpmd"
1592     elif u"l3fwd" in test_name:
1593         driver = u"l3fwd"
1594     elif u"avf" in test_name:
1595         driver = u"avf"
1596     elif u"rdma" in test_name:
1597         driver = u"rdma"
1598     elif u"dnv" in testbed or u"tsh" in testbed:
1599         driver = u"ixgbe"
1600     else:
1601         driver = u"i40e"
1602
1603     if u"acl" in test_name or \
1604             u"macip" in test_name or \
1605             u"nat" in test_name or \
1606             u"policer" in test_name or \
1607             u"cop" in test_name:
1608         bsf = u"features"
1609     elif u"scale" in test_name:
1610         bsf = u"scale"
1611     elif u"base" in test_name:
1612         bsf = u"base"
1613     else:
1614         bsf = u"base"
1615
1616     if u"114b" in test_name and u"vhost" in test_name:
1617         domain = u"vts"
1618     elif u"testpmd" in test_name or u"l3fwd" in test_name:
1619         domain = u"dpdk"
1620     elif u"memif" in test_name:
1621         domain = u"container_memif"
1622     elif u"srv6" in test_name:
1623         domain = u"srv6"
1624     elif u"vhost" in test_name:
1625         domain = u"vhost"
1626         if u"vppl2xc" in test_name:
1627             driver += u"-vpp"
1628         else:
1629             driver += u"-testpmd"
1630         if u"lbvpplacp" in test_name:
1631             bsf += u"-link-bonding"
1632     elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1633         domain = u"nf_service_density_vnfc"
1634     elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1635         domain = u"nf_service_density_cnfc"
1636     elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1637         domain = u"nf_service_density_cnfp"
1638     elif u"ipsec" in test_name:
1639         domain = u"ipsec"
1640         if u"sw" in test_name:
1641             bsf += u"-sw"
1642         elif u"hw" in test_name:
1643             bsf += u"-hw"
1644     elif u"ethip4vxlan" in test_name:
1645         domain = u"ip4_tunnels"
1646     elif u"ip4base" in test_name or u"ip4scale" in test_name:
1647         domain = u"ip4"
1648     elif u"ip6base" in test_name or u"ip6scale" in test_name:
1649         domain = u"ip6"
1650     elif u"l2xcbase" in test_name or \
1651             u"l2xcscale" in test_name or \
1652             u"l2bdbasemaclrn" in test_name or \
1653             u"l2bdscale" in test_name or \
1654             u"l2patch" in test_name:
1655         domain = u"l2"
1656     else:
1657         domain = u""
1658
1659     file_name = u"-".join((domain, testbed, nic)) + u".html#"
1660     anchor_name = u"-".join((frame_size, cores, bsf, driver))
1661
1662     return file_name + anchor_name
1663
1664
1665 def table_perf_trending_dash_html(table, input_data):
1666     """Generate the table(s) with algorithm:
1667     table_perf_trending_dash_html specified in the specification
1668     file.
1669
1670     :param table: Table to generate.
1671     :param input_data: Data to process.
1672     :type table: dict
1673     :type input_data: InputData
1674     """
1675
1676     _ = input_data
1677
1678     if not table.get(u"testbed", None):
1679         logging.error(
1680             f"The testbed is not defined for the table "
1681             f"{table.get(u'title', u'')}."
1682         )
1683         return
1684
1685     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1686
1687     try:
1688         with open(table[u"input-file"], u'rt') as csv_file:
1689             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1690     except KeyError:
1691         logging.warning(u"The input file is not defined.")
1692         return
1693     except csv.Error as err:
1694         logging.warning(
1695             f"Not possible to process the file {table[u'input-file']}.\n"
1696             f"{repr(err)}"
1697         )
1698         return
1699
1700     # Table:
1701     dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1702
1703     # Table header:
1704     trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1705     for idx, item in enumerate(csv_lst[0]):
1706         alignment = u"left" if idx == 0 else u"center"
1707         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1708         thead.text = item
1709
1710     # Rows:
1711     colors = {
1712         u"regression": (
1713             u"#ffcccc",
1714             u"#ff9999"
1715         ),
1716         u"progression": (
1717             u"#c6ecc6",
1718             u"#9fdf9f"
1719         ),
1720         u"normal": (
1721             u"#e9f1fb",
1722             u"#d4e4f7"
1723         )
1724     }
1725     for r_idx, row in enumerate(csv_lst[1:]):
1726         if int(row[4]):
1727             color = u"regression"
1728         elif int(row[5]):
1729             color = u"progression"
1730         else:
1731             color = u"normal"
1732         trow = ET.SubElement(
1733             dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
1734         )
1735
1736         # Columns:
1737         for c_idx, item in enumerate(row):
1738             tdata = ET.SubElement(
1739                 trow,
1740                 u"td",
1741                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1742             )
1743             # Name:
1744             if c_idx == 0:
1745                 ref = ET.SubElement(
1746                     tdata,
1747                     u"a",
1748                     attrib=dict(
1749                         href=f"../trending/"
1750                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1751                     )
1752                 )
1753                 ref.text = item
1754             else:
1755                 tdata.text = item
1756     try:
1757         with open(table[u"output-file"], u'w') as html_file:
1758             logging.info(f"    Writing file: {table[u'output-file']}")
1759             html_file.write(u".. raw:: html\n\n\t")
1760             html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
1761             html_file.write(u"\n\t<p><br><br></p>\n")
1762     except KeyError:
1763         logging.warning(u"The output file is not defined.")
1764         return
1765
1766
1767 def table_last_failed_tests(table, input_data):
1768     """Generate the table(s) with algorithm: table_last_failed_tests
1769     specified in the specification file.
1770
1771     :param table: Table to generate.
1772     :param input_data: Data to process.
1773     :type table: pandas.Series
1774     :type input_data: InputData
1775     """
1776
1777     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1778
1779     # Transform the data
1780     logging.info(
1781         f"    Creating the data set for the {table.get(u'type', u'')} "
1782         f"{table.get(u'title', u'')}."
1783     )
1784
1785     data = input_data.filter_data(table, continue_on_error=True)
1786
1787     if data is None or data.empty:
1788         logging.warning(
1789             f"    No data for the {table.get(u'type', u'')} "
1790             f"{table.get(u'title', u'')}."
1791         )
1792         return
1793
1794     tbl_list = list()
1795     for job, builds in table[u"data"].items():
1796         for build in builds:
1797             build = str(build)
1798             try:
1799                 version = input_data.metadata(job, build).get(u"version", u"")
1800             except KeyError:
1801                 logging.error(f"Data for {job}: {build} is not present.")
1802                 return
1803             tbl_list.append(build)
1804             tbl_list.append(version)
1805             failed_tests = list()
1806             passed = 0
1807             failed = 0
1808             for tst_data in data[job][build].values:
1809                 if tst_data[u"status"] != u"FAIL":
1810                     passed += 1
1811                     continue
1812                 failed += 1
1813                 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1814                 if not groups:
1815                     continue
1816                 nic = groups.group(0)
1817                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
1818             tbl_list.append(str(passed))
1819             tbl_list.append(str(failed))
1820             tbl_list.extend(failed_tests)
1821
1822     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1823     logging.info(f"    Writing file: {file_name}")
1824     with open(file_name, u"wt") as file_handler:
1825         for test in tbl_list:
1826             file_handler.write(test + u'\n')
1827
1828
1829 def table_failed_tests(table, input_data):
1830     """Generate the table(s) with algorithm: table_failed_tests
1831     specified in the specification file.
1832
1833     :param table: Table to generate.
1834     :param input_data: Data to process.
1835     :type table: pandas.Series
1836     :type input_data: InputData
1837     """
1838
1839     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1840
1841     # Transform the data
1842     logging.info(
1843         f"    Creating the data set for the {table.get(u'type', u'')} "
1844         f"{table.get(u'title', u'')}."
1845     )
1846     data = input_data.filter_data(table, continue_on_error=True)
1847
1848     # Prepare the header of the tables
1849     header = [
1850         u"Test Case",
1851         u"Failures [#]",
1852         u"Last Failure [Time]",
1853         u"Last Failure [VPP-Build-Id]",
1854         u"Last Failure [CSIT-Job-Build-Id]"
1855     ]
1856
1857     # Generate the data for the table according to the model in the table
1858     # specification
1859
1860     now = dt.utcnow()
1861     timeperiod = timedelta(int(table.get(u"window", 7)))
1862
1863     tbl_dict = dict()
1864     for job, builds in table[u"data"].items():
1865         for build in builds:
1866             build = str(build)
1867             for tst_name, tst_data in data[job][build].items():
1868                 if tst_name.lower() in table.get(u"ignore-list", list()):
1869                     continue
1870                 if tbl_dict.get(tst_name, None) is None:
1871                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1872                     if not groups:
1873                         continue
1874                     nic = groups.group(0)
1875                     tbl_dict[tst_name] = {
1876                         u"name": f"{nic}-{tst_data[u'name']}",
1877                         u"data": OrderedDict()
1878                     }
1879                 try:
1880                     generated = input_data.metadata(job, build).\
1881                         get(u"generated", u"")
1882                     if not generated:
1883                         continue
1884                     then = dt.strptime(generated, u"%Y%m%d %H:%M")
1885                     if (now - then) <= timeperiod:
1886                         tbl_dict[tst_name][u"data"][build] = (
1887                             tst_data[u"status"],
1888                             generated,
1889                             input_data.metadata(job, build).get(u"version",
1890                                                                 u""),
1891                             build
1892                         )
1893                 except (TypeError, KeyError) as err:
1894                     logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
1895
1896     max_fails = 0
1897     tbl_lst = list()
1898     for tst_data in tbl_dict.values():
1899         fails_nr = 0
1900         fails_last_date = u""
1901         fails_last_vpp = u""
1902         fails_last_csit = u""
1903         for val in tst_data[u"data"].values():
1904             if val[0] == u"FAIL":
1905                 fails_nr += 1
1906                 fails_last_date = val[1]
1907                 fails_last_vpp = val[2]
1908                 fails_last_csit = val[3]
1909         if fails_nr:
1910             max_fails = fails_nr if fails_nr > max_fails else max_fails
1911             tbl_lst.append(
1912                 [
1913                     tst_data[u"name"],
1914                     fails_nr,
1915                     fails_last_date,
1916                     fails_last_vpp,
1917                     f"mrr-daily-build-{fails_last_csit}"
1918                 ]
1919             )
1920
1921     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1922     tbl_sorted = list()
1923     for nrf in range(max_fails, -1, -1):
1924         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1925         tbl_sorted.extend(tbl_fails)
1926
1927     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1928     logging.info(f"    Writing file: {file_name}")
1929     with open(file_name, u"wt") as file_handler:
1930         file_handler.write(u",".join(header) + u"\n")
1931         for test in tbl_sorted:
1932             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1933
1934     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1935     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1936
1937
1938 def table_failed_tests_html(table, input_data):
1939     """Generate the table(s) with algorithm: table_failed_tests_html
1940     specified in the specification file.
1941
1942     :param table: Table to generate.
1943     :param input_data: Data to process.
1944     :type table: pandas.Series
1945     :type input_data: InputData
1946     """
1947
1948     _ = input_data
1949
1950     if not table.get(u"testbed", None):
1951         logging.error(
1952             f"The testbed is not defined for the table "
1953             f"{table.get(u'title', u'')}."
1954         )
1955         return
1956
1957     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1958
1959     try:
1960         with open(table[u"input-file"], u'rt') as csv_file:
1961             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1962     except KeyError:
1963         logging.warning(u"The input file is not defined.")
1964         return
1965     except csv.Error as err:
1966         logging.warning(
1967             f"Not possible to process the file {table[u'input-file']}.\n"
1968             f"{repr(err)}"
1969         )
1970         return
1971
1972     # Table:
1973     failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1974
1975     # Table header:
1976     trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1977     for idx, item in enumerate(csv_lst[0]):
1978         alignment = u"left" if idx == 0 else u"center"
1979         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1980         thead.text = item
1981
1982     # Rows:
1983     colors = (u"#e9f1fb", u"#d4e4f7")
1984     for r_idx, row in enumerate(csv_lst[1:]):
1985         background = colors[r_idx % 2]
1986         trow = ET.SubElement(
1987             failed_tests, u"tr", attrib=dict(bgcolor=background)
1988         )
1989
1990         # Columns:
1991         for c_idx, item in enumerate(row):
1992             tdata = ET.SubElement(
1993                 trow,
1994                 u"td",
1995                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1996             )
1997             # Name:
1998             if c_idx == 0:
1999                 ref = ET.SubElement(
2000                     tdata,
2001                     u"a",
2002                     attrib=dict(
2003                         href=f"../trending/"
2004                              f"{_generate_url(table.get(u'testbed', ''), item)}"
2005                     )
2006                 )
2007                 ref.text = item
2008             else:
2009                 tdata.text = item
2010     try:
2011         with open(table[u"output-file"], u'w') as html_file:
2012             logging.info(f"    Writing file: {table[u'output-file']}")
2013             html_file.write(u".. raw:: html\n\n\t")
2014             html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
2015             html_file.write(u"\n\t<p><br><br></p>\n")
2016     except KeyError:
2017         logging.warning(u"The output file is not defined.")
2018         return