resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from collections import OrderedDict
  23 from xml.etree import ElementTree as ET
  24 from datetime import datetime as dt
  25 from datetime import timedelta
  26
  27 import plotly.graph_objects as go
  28 import plotly.offline as ploff
  29 import pandas as pd
  30
  31 from numpy import nan, isnan
  32 from yaml import load, FullLoader, YAMLError
  33
  34 from pal_utils import mean, stdev, classify_anomalies, \
  35     convert_csv_to_pretty_txt, relative_change_stdev
  36
  37
  38 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
  39
  40
  41 def generate_tables(spec, data):
  42     """Generate all tables specified in the specification file.
  43
  44     :param spec: Specification read from the specification file.
  45     :param data: Data to process.
  46     :type spec: Specification
  47     :type data: InputData
  48     """
  49
  50     generator = {
  51         u"table_merged_details": table_merged_details,
  52         u"table_perf_comparison": table_perf_comparison,
  53         u"table_perf_comparison_nic": table_perf_comparison_nic,
  54         u"table_nics_comparison": table_nics_comparison,
  55         u"table_soak_vs_ndr": table_soak_vs_ndr,
  56         u"table_perf_trending_dash": table_perf_trending_dash,
  57         u"table_perf_trending_dash_html": table_perf_trending_dash_html,
  58         u"table_last_failed_tests": table_last_failed_tests,
  59         u"table_failed_tests": table_failed_tests,
  60         u"table_failed_tests_html": table_failed_tests_html,
  61         u"table_oper_data_html": table_oper_data_html
  62     }
  63
  64     logging.info(u"Generating the tables ...")
  65     for table in spec.tables:
  66         try:
  67             generator[table[u"algorithm"]](table, data)
  68         except NameError as err:
  69             logging.error(
  70                 f"Probably algorithm {table[u'algorithm']} is not defined: "
  71                 f"{repr(err)}"
  72             )
  73     logging.info(u"Done.")
  74
  75
  76 def table_oper_data_html(table, input_data):
  77     """Generate the table(s) with algorithm: html_table_oper_data
  78     specified in the specification file.
  79
  80     :param table: Table to generate.
  81     :param input_data: Data to process.
  82     :type table: pandas.Series
  83     :type input_data: InputData
  84     """
  85
  86     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
  87     # Transform the data
  88     logging.info(
  89         f"    Creating the data set for the {table.get(u'type', u'')} "
  90         f"{table.get(u'title', u'')}."
  91     )
  92     data = input_data.filter_data(
  93         table,
  94         params=[u"name", u"parent", u"show-run", u"type"],
  95         continue_on_error=True
  96     )
  97     if data.empty:
  98         return
  99     data = input_data.merge_data(data)
 100
 101     sort_tests = table.get(u"sort", None)
 102     if sort_tests:
 103         args = dict(
 104             inplace=True,
 105             ascending=(sort_tests == u"ascending")
 106         )
 107         data.sort_index(**args)
 108
 109     suites = input_data.filter_data(
 110         table,
 111         continue_on_error=True,
 112         data_set=u"suites"
 113     )
 114     if suites.empty:
 115         return
 116     suites = input_data.merge_data(suites)
 117
 118     def _generate_html_table(tst_data):
 119         """Generate an HTML table with operational data for the given test.
 120
 121         :param tst_data: Test data to be used to generate the table.
 122         :type tst_data: pandas.Series
 123         :returns: HTML table with operational data.
 124         :rtype: str
 125         """
 126
 127         colors = {
 128             u"header": u"#7eade7",
 129             u"empty": u"#ffffff",
 130             u"body": (u"#e9f1fb", u"#d4e4f7")
 131         }
 132
 133         tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
 134
 135         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
 136         thead = ET.SubElement(
 137             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 138         )
 139         thead.text = tst_data[u"name"]
 140
 141         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 142         thead = ET.SubElement(
 143             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 144         )
 145         thead.text = u"\t"
 146
 147         if tst_data.get(u"show-run", u"No Data") == u"No Data":
 148             trow = ET.SubElement(
 149                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 150             )
 151             tcol = ET.SubElement(
 152                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 153             )
 154             tcol.text = u"No Data"
 155
 156             trow = ET.SubElement(
 157                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 158             )
 159             thead = ET.SubElement(
 160                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 161             )
 162             font = ET.SubElement(
 163                 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 164             )
 165             font.text = u"."
 166             return str(ET.tostring(tbl, encoding=u"unicode"))
 167
 168         tbl_hdr = (
 169             u"Name",
 170             u"Nr of Vectors",
 171             u"Nr of Packets",
 172             u"Suspends",
 173             u"Cycles per Packet",
 174             u"Average Vector Size"
 175         )
 176
 177         for dut_data in tst_data[u"show-run"].values():
 178             trow = ET.SubElement(
 179                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 180             )
 181             tcol = ET.SubElement(
 182                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 183             )
 184             if dut_data.get(u"threads", None) is None:
 185                 tcol.text = u"No Data"
 186                 continue
 187
 188             bold = ET.SubElement(tcol, u"b")
 189             bold.text = (
 190                 f"Host IP: {dut_data.get(u'host', '')}, "
 191                 f"Socket: {dut_data.get(u'socket', '')}"
 192             )
 193             trow = ET.SubElement(
 194                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 195             )
 196             thead = ET.SubElement(
 197                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 198             )
 199             thead.text = u"\t"
 200
 201             for thread_nr, thread in dut_data[u"threads"].items():
 202                 trow = ET.SubElement(
 203                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 204                 )
 205                 tcol = ET.SubElement(
 206                     trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 207                 )
 208                 bold = ET.SubElement(tcol, u"b")
 209                 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
 210                 trow = ET.SubElement(
 211                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 212                 )
 213                 for idx, col in enumerate(tbl_hdr):
 214                     tcol = ET.SubElement(
 215                         trow, u"td",
 216                         attrib=dict(align=u"right" if idx else u"left")
 217                     )
 218                     font = ET.SubElement(
 219                         tcol, u"font", attrib=dict(size=u"2")
 220                     )
 221                     bold = ET.SubElement(font, u"b")
 222                     bold.text = col
 223                 for row_nr, row in enumerate(thread):
 224                     trow = ET.SubElement(
 225                         tbl, u"tr",
 226                         attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
 227                     )
 228                     for idx, col in enumerate(row):
 229                         tcol = ET.SubElement(
 230                             trow, u"td",
 231                             attrib=dict(align=u"right" if idx else u"left")
 232                         )
 233                         font = ET.SubElement(
 234                             tcol, u"font", attrib=dict(size=u"2")
 235                         )
 236                         if isinstance(col, float):
 237                             font.text = f"{col:.2f}"
 238                         else:
 239                             font.text = str(col)
 240                 trow = ET.SubElement(
 241                     tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 242                 )
 243                 thead = ET.SubElement(
 244                     trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 245                 )
 246                 thead.text = u"\t"
 247
 248         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 249         thead = ET.SubElement(
 250             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 251         )
 252         font = ET.SubElement(
 253             thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 254         )
 255         font.text = u"."
 256
 257         return str(ET.tostring(tbl, encoding=u"unicode"))
 258
 259     for suite in suites.values:
 260         html_table = str()
 261         for test_data in data.values:
 262             if test_data[u"parent"] not in suite[u"name"]:
 263                 continue
 264             html_table += _generate_html_table(test_data)
 265         if not html_table:
 266             continue
 267         try:
 268             file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
 269             with open(f"{file_name}", u'w') as html_file:
 270                 logging.info(f"    Writing file: {file_name}")
 271                 html_file.write(u".. raw:: html\n\n\t")
 272                 html_file.write(html_table)
 273                 html_file.write(u"\n\t<p><br><br></p>\n")
 274         except KeyError:
 275             logging.warning(u"The output file is not defined.")
 276             return
 277     logging.info(u"  Done.")
 278
 279
 280 def table_merged_details(table, input_data):
 281     """Generate the table(s) with algorithm: table_merged_details
 282     specified in the specification file.
 283
 284     :param table: Table to generate.
 285     :param input_data: Data to process.
 286     :type table: pandas.Series
 287     :type input_data: InputData
 288     """
 289
 290     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 291
 292     # Transform the data
 293     logging.info(
 294         f"    Creating the data set for the {table.get(u'type', u'')} "
 295         f"{table.get(u'title', u'')}."
 296     )
 297     data = input_data.filter_data(table, continue_on_error=True)
 298     data = input_data.merge_data(data)
 299
 300     sort_tests = table.get(u"sort", None)
 301     if sort_tests:
 302         args = dict(
 303             inplace=True,
 304             ascending=(sort_tests == u"ascending")
 305         )
 306         data.sort_index(**args)
 307
 308     suites = input_data.filter_data(
 309         table, continue_on_error=True, data_set=u"suites")
 310     suites = input_data.merge_data(suites)
 311
 312     # Prepare the header of the tables
 313     header = list()
 314     for column in table[u"columns"]:
 315         header.append(
 316             u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
 317         )
 318
 319     for suite in suites.values:
 320         # Generate data
 321         suite_name = suite[u"name"]
 322         table_lst = list()
 323         for test in data.keys():
 324             if data[test][u"parent"] not in suite_name:
 325                 continue
 326             row_lst = list()
 327             for column in table[u"columns"]:
 328                 try:
 329                     col_data = str(data[test][column[
 330                         u"data"].split(u" ")[1]]).replace(u'"', u'""')
 331                     # Do not include tests with "Test Failed" in test message
 332                     if u"Test Failed" in col_data:
 333                         continue
 334                     col_data = col_data.replace(
 335                         u"No Data", u"Not Captured     "
 336                     )
 337                     if column[u"data"].split(u" ")[1] in (u"name", ):
 338                         if len(col_data) > 30:
 339                             col_data_lst = col_data.split(u"-")
 340                             half = int(len(col_data_lst) / 2)
 341                             col_data = f"{u'-'.join(col_data_lst[:half])}" \
 342                                        f"- |br| " \
 343                                        f"{u'-'.join(col_data_lst[half:])}"
 344                         col_data = f" |prein| {col_data} |preout| "
 345                     elif column[u"data"].split(u" ")[1] in (u"msg", ):
 346                         # Temporary solution: remove NDR results from message:
 347                         if bool(table.get(u'remove-ndr', False)):
 348                             try:
 349                                 col_data = col_data.split(u" |br| ", 1)[1]
 350                             except IndexError:
 351                                 pass
 352                         col_data = f" |prein| {col_data} |preout| "
 353                     elif column[u"data"].split(u" ")[1] in \
 354                             (u"conf-history", u"show-run"):
 355                         col_data = col_data.replace(u" |br| ", u"", 1)
 356                         col_data = f" |prein| {col_data[:-5]} |preout| "
 357                     row_lst.append(f'"{col_data}"')
 358                 except KeyError:
 359                     row_lst.append(u'"Not captured"')
 360             if len(row_lst) == len(table[u"columns"]):
 361                 table_lst.append(row_lst)
 362
 363         # Write the data to file
 364         if table_lst:
 365             separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
 366             file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
 367             logging.info(f"      Writing file: {file_name}")
 368             with open(file_name, u"wt") as file_handler:
 369                 file_handler.write(u",".join(header) + u"\n")
 370                 for item in table_lst:
 371                     file_handler.write(u",".join(item) + u"\n")
 372
 373     logging.info(u"  Done.")
 374
 375
 376 def _tpc_modify_test_name(test_name):
 377     """Modify a test name by replacing its parts.
 378
 379     :param test_name: Test name to be modified.
 380     :type test_name: str
 381     :returns: Modified test name.
 382     :rtype: str
 383     """
 384     test_name_mod = test_name.\
 385         replace(u"-ndrpdrdisc", u""). \
 386         replace(u"-ndrpdr", u"").\
 387         replace(u"-pdrdisc", u""). \
 388         replace(u"-ndrdisc", u"").\
 389         replace(u"-pdr", u""). \
 390         replace(u"-ndr", u""). \
 391         replace(u"1t1c", u"1c").\
 392         replace(u"2t1c", u"1c"). \
 393         replace(u"2t2c", u"2c").\
 394         replace(u"4t2c", u"2c"). \
 395         replace(u"4t4c", u"4c").\
 396         replace(u"8t4c", u"4c")
 397
 398     return re.sub(REGEX_NIC, u"", test_name_mod)
 399
 400
 401 def _tpc_modify_displayed_test_name(test_name):
 402     """Modify a test name which is displayed in a table by replacing its parts.
 403
 404     :param test_name: Test name to be modified.
 405     :type test_name: str
 406     :returns: Modified test name.
 407     :rtype: str
 408     """
 409     return test_name.\
 410         replace(u"1t1c", u"1c").\
 411         replace(u"2t1c", u"1c"). \
 412         replace(u"2t2c", u"2c").\
 413         replace(u"4t2c", u"2c"). \
 414         replace(u"4t4c", u"4c").\
 415         replace(u"8t4c", u"4c")
 416
 417
 418 def _tpc_insert_data(target, src, include_tests):
 419     """Insert src data to the target structure.
 420
 421     :param target: Target structure where the data is placed.
 422     :param src: Source data to be placed into the target stucture.
 423     :param include_tests: Which results will be included (MRR, NDR, PDR).
 424     :type target: list
 425     :type src: dict
 426     :type include_tests: str
 427     """
 428     try:
 429         if include_tests == u"MRR":
 430             target.append(
 431                 (
 432                     src[u"result"][u"receive-rate"],
 433                     src[u"result"][u"receive-stdev"]
 434                 )
 435             )
 436         elif include_tests == u"PDR":
 437             target.append(src[u"throughput"][u"PDR"][u"LOWER"])
 438         elif include_tests == u"NDR":
 439             target.append(src[u"throughput"][u"NDR"][u"LOWER"])
 440     except (KeyError, TypeError):
 441         pass
 442
 443
 444 def _tpc_sort_table(table):
 445     """Sort the table this way:
 446
 447     1. Put "New in CSIT-XXXX" at the first place.
 448     2. Put "See footnote" at the second place.
 449     3. Sort the rest by "Delta".
 450
 451     :param table: Table to sort.
 452     :type table: list
 453     :returns: Sorted table.
 454     :rtype: list
 455     """
 456
 457     tbl_new = list()
 458     tbl_see = list()
 459     tbl_delta = list()
 460     for item in table:
 461         if isinstance(item[-1], str):
 462             if u"New in CSIT" in item[-1]:
 463                 tbl_new.append(item)
 464             elif u"See footnote" in item[-1]:
 465                 tbl_see.append(item)
 466         else:
 467             tbl_delta.append(item)
 468
 469     # Sort the tables:
 470     tbl_new.sort(key=lambda rel: rel[0], reverse=False)
 471     tbl_see.sort(key=lambda rel: rel[0], reverse=False)
 472     tbl_see.sort(key=lambda rel: rel[-2], reverse=False)
 473     tbl_delta.sort(key=lambda rel: rel[0], reverse=False)
 474     tbl_delta.sort(key=lambda rel: rel[-2], reverse=True)
 475
 476     # Put the tables together:
 477     table = list()
 478     # We do not want "New in CSIT":
 479     # table.extend(tbl_new)
 480     table.extend(tbl_see)
 481     table.extend(tbl_delta)
 482
 483     return table
 484
 485
 486 def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
 487                              footnote=u""):
 488     """Generate html table from input data with simple sorting possibility.
 489
 490     :param header: Table header.
 491     :param data: Input data to be included in the table. It is a list of lists.
 492         Inner lists are rows in the table. All inner lists must be of the same
 493         length. The length of these lists must be the same as the length of the
 494         header.
 495     :param out_file_name: The name (relative or full path) where the
 496         generated html table is written.
 497     :param legend: The legend to display below the table.
 498     :param footnote: The footnote to display below the table (and legend).
 499     :type header: list
 500     :type data: list of lists
 501     :type out_file_name: str
 502     :type legend: str
 503     :type footnote: str
 504     """
 505
 506     try:
 507         idx = header.index(u"Test Case")
 508     except ValueError:
 509         idx = 0
 510     params = {
 511         u"align-hdr": ([u"left", u"center"], [u"left", u"left", u"center"]),
 512         u"align-itm": ([u"left", u"right"], [u"left", u"left", u"right"]),
 513         u"width": ([28, 9], [4, 24, 10])
 514     }
 515
 516     df_data = pd.DataFrame(data, columns=header)
 517
 518     df_sorted = [df_data.sort_values(
 519         by=[key, header[idx]], ascending=[True, True]
 520         if key != header[idx] else [False, True]) for key in header]
 521     df_sorted_rev = [df_data.sort_values(
 522         by=[key, header[idx]], ascending=[False, True]
 523         if key != header[idx] else [True, True]) for key in header]
 524     df_sorted.extend(df_sorted_rev)
 525
 526     fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
 527                    for idx in range(len(df_data))]]
 528     table_header = dict(
 529         values=[f"<b>{item}</b>" for item in header],
 530         fill_color=u"#7eade7",
 531         align=params[u"align-hdr"][idx]
 532     )
 533
 534     fig = go.Figure()
 535
 536     for table in df_sorted:
 537         columns = [table.get(col) for col in header]
 538         fig.add_trace(
 539             go.Table(
 540                 columnwidth=params[u"width"][idx],
 541                 header=table_header,
 542                 cells=dict(
 543                     values=columns,
 544                     fill_color=fill_color,
 545                     align=params[u"align-itm"][idx]
 546                 )
 547             )
 548         )
 549
 550     buttons = list()
 551     menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
 552     menu_items_rev = [f"<b>{itm}</b> (descending)" for itm in header]
 553     menu_items.extend(menu_items_rev)
 554     for idx, hdr in enumerate(menu_items):
 555         visible = [False, ] * len(menu_items)
 556         visible[idx] = True
 557         buttons.append(
 558             dict(
 559                 label=hdr.replace(u" [Mpps]", u""),
 560                 method=u"update",
 561                 args=[{u"visible": visible}],
 562             )
 563         )
 564
 565     fig.update_layout(
 566         updatemenus=[
 567             go.layout.Updatemenu(
 568                 type=u"dropdown",
 569                 direction=u"down",
 570                 x=0.0,  # 0.03,
 571                 xanchor=u"left",
 572                 y=1.045,
 573                 yanchor=u"top",
 574                 active=len(menu_items) - 1,
 575                 buttons=list(buttons)
 576             )
 577         ],
 578         # annotations=[
 579         #     go.layout.Annotation(
 580         #         text=u"<b>Sort by:</b>",
 581         #         x=0,
 582         #         xref=u"paper",
 583         #         y=1.035,
 584         #         yref=u"paper",
 585         #         align=u"left",
 586         #         showarrow=False
 587         #     )
 588         # ]
 589     )
 590
 591     ploff.plot(
 592         fig,
 593         show_link=False,
 594         auto_open=False,
 595         filename=f"{out_file_name}_in.html"
 596     )
 597
 598     file_name = out_file_name.split(u"/")[-1]
 599     if u"vpp" in out_file_name:
 600         path = u"_tmp/src/vpp_performance_tests/comparisons/"
 601     else:
 602         path = u"_tmp/src/dpdk_performance_tests/comparisons/"
 603     with open(f"{path}{file_name}.rst", u"wt") as rst_file:
 604         rst_file.write(
 605             u"\n"
 606             u".. |br| raw:: html\n\n    <br />\n\n\n"
 607             u".. |prein| raw:: html\n\n    <pre>\n\n\n"
 608             u".. |preout| raw:: html\n\n    </pre>\n\n"
 609         )
 610         rst_file.write(
 611             u".. raw:: html\n\n"
 612             f'    <iframe frameborder="0" scrolling="no" '
 613             f'width="1600" height="1000" '
 614             f'src="../..{out_file_name.replace(u"_build", u"")}_in.html">'
 615             f'</iframe>\n\n'
 616         )
 617         if legend:
 618             rst_file.write(legend[1:].replace(u"\n", u" |br| "))
 619         if footnote:
 620             rst_file.write(footnote.replace(u"\n", u" |br| ")[1:])
 621
 622
 623 def table_perf_comparison(table, input_data):
 624     """Generate the table(s) with algorithm: table_perf_comparison
 625     specified in the specification file.
 626
 627     :param table: Table to generate.
 628     :param input_data: Data to process.
 629     :type table: pandas.Series
 630     :type input_data: InputData
 631     """
 632
 633     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 634
 635     # Transform the data
 636     logging.info(
 637         f"    Creating the data set for the {table.get(u'type', u'')} "
 638         f"{table.get(u'title', u'')}."
 639     )
 640     data = input_data.filter_data(table, continue_on_error=True)
 641
 642     # Prepare the header of the tables
 643     try:
 644         header = [u"Test Case", ]
 645         legend = u"\nLegend:\n"
 646
 647         rca_data = None
 648         rca = table.get(u"rca", None)
 649         if rca:
 650             try:
 651                 with open(rca.get(u"data-file", ""), u"r") as rca_file:
 652                     rca_data = load(rca_file, Loader=FullLoader)
 653                 header.insert(0, rca.get(u"title", "RCA"))
 654                 legend += (
 655                     u"RCA: Reference to the Root Cause Analysis, see below.\n"
 656                 )
 657             except (YAMLError, IOError) as err:
 658                 logging.warning(repr(err))
 659
 660         history = table.get(u"history", list())
 661         for item in history:
 662             header.extend(
 663                 [
 664                     f"{item[u'title']} Avg({table[u'include-tests']})",
 665                     f"{item[u'title']} Stdev({table[u'include-tests']})"
 666                 ]
 667             )
 668             legend += (
 669                 f"{item[u'title']} Avg({table[u'include-tests']}): "
 670                 f"Mean value of {table[u'include-tests']} [Mpps] computed from "
 671                 f"a series of runs of the listed tests executed against "
 672                 f"{item[u'title']}.\n"
 673                 f"{item[u'title']} Stdev({table[u'include-tests']}): "
 674                 f"Standard deviation value of {table[u'include-tests']} [Mpps] "
 675                 f"computed from a series of runs of the listed tests executed "
 676                 f"against {item[u'title']}.\n"
 677             )
 678         header.extend(
 679             [
 680                 f"{table[u'reference'][u'title']} "
 681                 f"Avg({table[u'include-tests']})",
 682                 f"{table[u'reference'][u'title']} "
 683                 f"Stdev({table[u'include-tests']})",
 684                 f"{table[u'compare'][u'title']} "
 685                 f"Avg({table[u'include-tests']})",
 686                 f"{table[u'compare'][u'title']} "
 687                 f"Stdev({table[u'include-tests']})",
 688                 f"Diff({table[u'reference'][u'title']},"
 689                 f"{table[u'compare'][u'title']})",
 690                 u"Stdev(Diff)"
 691             ]
 692         )
 693         header_str = u";".join(header) + u"\n"
 694         legend += (
 695             f"{table[u'reference'][u'title']} "
 696             f"Avg({table[u'include-tests']}): "
 697             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
 698             f"series of runs of the listed tests executed against "
 699             f"{table[u'reference'][u'title']}.\n"
 700             f"{table[u'reference'][u'title']} "
 701             f"Stdev({table[u'include-tests']}): "
 702             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
 703             f"computed from a series of runs of the listed tests executed "
 704             f"against {table[u'reference'][u'title']}.\n"
 705             f"{table[u'compare'][u'title']} "
 706             f"Avg({table[u'include-tests']}): "
 707             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
 708             f"series of runs of the listed tests executed against "
 709             f"{table[u'compare'][u'title']}.\n"
 710             f"{table[u'compare'][u'title']} "
 711             f"Stdev({table[u'include-tests']}): "
 712             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
 713             f"computed from a series of runs of the listed tests executed "
 714             f"against {table[u'compare'][u'title']}.\n"
 715             f"Diff({table[u'reference'][u'title']},"
 716             f"{table[u'compare'][u'title']}): "
 717             f"Percentage change calculated for mean values.\n"
 718             u"Stdev(Diff): "
 719             u"Standard deviation of percentage change calculated for mean "
 720             u"values.\n"
 721             u"NT: Not Tested\n"
 722         )
 723     except (AttributeError, KeyError) as err:
 724         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 725         return
 726
 727     # Prepare data to the table:
 728     tbl_dict = dict()
 729     for job, builds in table[u"reference"][u"data"].items():
 730         for build in builds:
 731             for tst_name, tst_data in data[job][str(build)].items():
 732                 tst_name_mod = _tpc_modify_test_name(tst_name)
 733                 if (u"across topologies" in table[u"title"].lower() or
 734                         (u" 3n-" in table[u"title"].lower() and
 735                          u" 2n-" in table[u"title"].lower())):
 736                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 737                 if tbl_dict.get(tst_name_mod, None) is None:
 738                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
 739                     nic = groups.group(0) if groups else u""
 740                     name = \
 741                         f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 742                     if u"across testbeds" in table[u"title"].lower() or \
 743                             u"across topologies" in table[u"title"].lower():
 744                         name = _tpc_modify_displayed_test_name(name)
 745                     tbl_dict[tst_name_mod] = {
 746                         u"name": name,
 747                         u"ref-data": list(),
 748                         u"cmp-data": list()
 749                     }
 750                 _tpc_insert_data(target=tbl_dict[tst_name_mod][u"ref-data"],
 751                                  src=tst_data,
 752                                  include_tests=table[u"include-tests"])
 753
 754     replacement = table[u"reference"].get(u"data-replacement", None)
 755     if replacement:
 756         create_new_list = True
 757         rpl_data = input_data.filter_data(
 758             table, data=replacement, continue_on_error=True)
 759         for job, builds in replacement.items():
 760             for build in builds:
 761                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 762                     tst_name_mod = _tpc_modify_test_name(tst_name)
 763                     if (u"across topologies" in table[u"title"].lower() or
 764                             (u" 3n-" in table[u"title"].lower() and
 765                              u" 2n-" in table[u"title"].lower())):
 766                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 767                     if tbl_dict.get(tst_name_mod, None) is None:
 768                         name = \
 769                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 770                         if u"across testbeds" in table[u"title"].lower() or \
 771                                 u"across topologies" in table[u"title"].lower():
 772                             name = _tpc_modify_displayed_test_name(name)
 773                         tbl_dict[tst_name_mod] = {
 774                             u"name": name,
 775                             u"ref-data": list(),
 776                             u"cmp-data": list()
 777                         }
 778                     if create_new_list:
 779                         create_new_list = False
 780                         tbl_dict[tst_name_mod][u"ref-data"] = list()
 781
 782                     _tpc_insert_data(
 783                         target=tbl_dict[tst_name_mod][u"ref-data"],
 784                         src=tst_data,
 785                         include_tests=table[u"include-tests"]
 786                     )
 787
 788     for job, builds in table[u"compare"][u"data"].items():
 789         for build in builds:
 790             for tst_name, tst_data in data[job][str(build)].items():
 791                 tst_name_mod = _tpc_modify_test_name(tst_name)
 792                 if (u"across topologies" in table[u"title"].lower() or
 793                         (u" 3n-" in table[u"title"].lower() and
 794                          u" 2n-" in table[u"title"].lower())):
 795                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 796                 if tbl_dict.get(tst_name_mod, None) is None:
 797                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
 798                     nic = groups.group(0) if groups else u""
 799                     name = \
 800                         f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 801                     if u"across testbeds" in table[u"title"].lower() or \
 802                             u"across topologies" in table[u"title"].lower():
 803                         name = _tpc_modify_displayed_test_name(name)
 804                     tbl_dict[tst_name_mod] = {
 805                         u"name": name,
 806                         u"ref-data": list(),
 807                         u"cmp-data": list()
 808                     }
 809                 _tpc_insert_data(
 810                     target=tbl_dict[tst_name_mod][u"cmp-data"],
 811                     src=tst_data,
 812                     include_tests=table[u"include-tests"]
 813                 )
 814
 815     replacement = table[u"compare"].get(u"data-replacement", None)
 816     if replacement:
 817         create_new_list = True
 818         rpl_data = input_data.filter_data(
 819             table, data=replacement, continue_on_error=True)
 820         for job, builds in replacement.items():
 821             for build in builds:
 822                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 823                     tst_name_mod = _tpc_modify_test_name(tst_name)
 824                     if (u"across topologies" in table[u"title"].lower() or
 825                             (u" 3n-" in table[u"title"].lower() and
 826                              u" 2n-" in table[u"title"].lower())):
 827                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 828                     if tbl_dict.get(tst_name_mod, None) is None:
 829                         name = \
 830                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 831                         if u"across testbeds" in table[u"title"].lower() or \
 832                                 u"across topologies" in table[u"title"].lower():
 833                             name = _tpc_modify_displayed_test_name(name)
 834                         tbl_dict[tst_name_mod] = {
 835                             u"name": name,
 836                             u"ref-data": list(),
 837                             u"cmp-data": list()
 838                         }
 839                     if create_new_list:
 840                         create_new_list = False
 841                         tbl_dict[tst_name_mod][u"cmp-data"] = list()
 842
 843                     _tpc_insert_data(
 844                         target=tbl_dict[tst_name_mod][u"cmp-data"],
 845                         src=tst_data,
 846                         include_tests=table[u"include-tests"]
 847                     )
 848
 849     for item in history:
 850         for job, builds in item[u"data"].items():
 851             for build in builds:
 852                 for tst_name, tst_data in data[job][str(build)].items():
 853                     tst_name_mod = _tpc_modify_test_name(tst_name)
 854                     if (u"across topologies" in table[u"title"].lower() or
 855                             (u" 3n-" in table[u"title"].lower() and
 856                              u" 2n-" in table[u"title"].lower())):
 857                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 858                     if tbl_dict.get(tst_name_mod, None) is None:
 859                         continue
 860                     if tbl_dict[tst_name_mod].get(u"history", None) is None:
 861                         tbl_dict[tst_name_mod][u"history"] = OrderedDict()
 862                     if tbl_dict[tst_name_mod][u"history"].\
 863                             get(item[u"title"], None) is None:
 864                         tbl_dict[tst_name_mod][u"history"][item[
 865                             u"title"]] = list()
 866                     try:
 867                         if table[u"include-tests"] == u"MRR":
 868                             res = (tst_data[u"result"][u"receive-rate"],
 869                                    tst_data[u"result"][u"receive-stdev"])
 870                         elif table[u"include-tests"] == u"PDR":
 871                             res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
 872                         elif table[u"include-tests"] == u"NDR":
 873                             res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
 874                         else:
 875                             continue
 876                         tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
 877                             append(res)
 878                     except (TypeError, KeyError):
 879                         pass
 880
 881     tbl_lst = list()
 882     for tst_name in tbl_dict:
 883         item = [tbl_dict[tst_name][u"name"], ]
 884         if history:
 885             if tbl_dict[tst_name].get(u"history", None) is not None:
 886                 for hist_data in tbl_dict[tst_name][u"history"].values():
 887                     if hist_data:
 888                         if table[u"include-tests"] == u"MRR":
 889                             item.append(round(hist_data[0][0] / 1e6, 1))
 890                             item.append(round(hist_data[0][1] / 1e6, 1))
 891                         else:
 892                             item.append(round(mean(hist_data) / 1e6, 1))
 893                             item.append(round(stdev(hist_data) / 1e6, 1))
 894                     else:
 895                         item.extend([u"NT", u"NT"])
 896             else:
 897                 item.extend([u"NT", u"NT"])
 898         data_r = tbl_dict[tst_name][u"ref-data"]
 899         if data_r:
 900             if table[u"include-tests"] == u"MRR":
 901                 data_r_mean = data_r[0][0]
 902                 data_r_stdev = data_r[0][1]
 903             else:
 904                 data_r_mean = mean(data_r)
 905                 data_r_stdev = stdev(data_r)
 906             item.append(round(data_r_mean / 1e6, 1))
 907             item.append(round(data_r_stdev / 1e6, 1))
 908         else:
 909             data_r_mean = None
 910             data_r_stdev = None
 911             item.extend([u"NT", u"NT"])
 912         data_c = tbl_dict[tst_name][u"cmp-data"]
 913         if data_c:
 914             if table[u"include-tests"] == u"MRR":
 915                 data_c_mean = data_c[0][0]
 916                 data_c_stdev = data_c[0][1]
 917             else:
 918                 data_c_mean = mean(data_c)
 919                 data_c_stdev = stdev(data_c)
 920             item.append(round(data_c_mean / 1e6, 1))
 921             item.append(round(data_c_stdev / 1e6, 1))
 922         else:
 923             data_c_mean = None
 924             data_c_stdev = None
 925             item.extend([u"NT", u"NT"])
 926         if item[-2] == u"NT":
 927             pass
 928         elif item[-4] == u"NT":
 929             item.append(u"New in CSIT-2001")
 930             item.append(u"New in CSIT-2001")
 931         elif data_r_mean is not None and data_c_mean is not None:
 932             delta, d_stdev = relative_change_stdev(
 933                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
 934             )
 935             try:
 936                 item.append(round(delta))
 937             except ValueError:
 938                 item.append(delta)
 939             try:
 940                 item.append(round(d_stdev))
 941             except ValueError:
 942                 item.append(d_stdev)
 943         if rca_data:
 944             rca_nr = rca_data.get(item[0], u"-")
 945             item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-")
 946         if (len(item) == len(header)) and (item[-4] != u"NT"):
 947             tbl_lst.append(item)
 948
 949     tbl_lst = _tpc_sort_table(tbl_lst)
 950
 951     # Generate csv tables:
 952     csv_file = f"{table[u'output-file']}.csv"
 953     with open(csv_file, u"wt") as file_handler:
 954         file_handler.write(header_str)
 955         for test in tbl_lst:
 956             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 957
 958     txt_file_name = f"{table[u'output-file']}.txt"
 959     convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
 960
 961     footnote = u""
 962     with open(txt_file_name, u'a') as txt_file:
 963         txt_file.write(legend)
 964         if rca_data:
 965             footnote = rca_data.get(u"footnote", u"")
 966             if footnote:
 967                 txt_file.write(footnote)
 968         txt_file.write(u":END")
 969
 970     # Generate html table:
 971     _tpc_generate_html_table(
 972         header,
 973         tbl_lst,
 974         table[u'output-file'],
 975         legend=legend,
 976         footnote=footnote
 977     )
 978
 979
 980 def table_perf_comparison_nic(table, input_data):
 981     """Generate the table(s) with algorithm: table_perf_comparison
 982     specified in the specification file.
 983
 984     :param table: Table to generate.
 985     :param input_data: Data to process.
 986     :type table: pandas.Series
 987     :type input_data: InputData
 988     """
 989
 990     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 991
 992     # Transform the data
 993     logging.info(
 994         f"    Creating the data set for the {table.get(u'type', u'')} "
 995         f"{table.get(u'title', u'')}."
 996     )
 997     data = input_data.filter_data(table, continue_on_error=True)
 998
 999     # Prepare the header of the tables
1000     try:
1001         header = [u"Test Case", ]
1002         legend = u"\nLegend:\n"
1003
1004         rca_data = None
1005         rca = table.get(u"rca", None)
1006         if rca:
1007             try:
1008                 with open(rca.get(u"data-file", ""), u"r") as rca_file:
1009                     rca_data = load(rca_file, Loader=FullLoader)
1010                 header.insert(0, rca.get(u"title", "RCA"))
1011                 legend += (
1012                     u"RCA: Reference to the Root Cause Analysis, see below.\n"
1013                 )
1014             except (YAMLError, IOError) as err:
1015                 logging.warning(repr(err))
1016
1017         history = table.get(u"history", list())
1018         for item in history:
1019             header.extend(
1020                 [
1021                     f"{item[u'title']} Avg({table[u'include-tests']})",
1022                     f"{item[u'title']} Stdev({table[u'include-tests']})"
1023                 ]
1024             )
1025             legend += (
1026                 f"{item[u'title']} Avg({table[u'include-tests']}): "
1027                 f"Mean value of {table[u'include-tests']} [Mpps] computed from "
1028                 f"a series of runs of the listed tests executed against "
1029                 f"{item[u'title']}.\n"
1030                 f"{item[u'title']} Stdev({table[u'include-tests']}): "
1031                 f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1032                 f"computed from a series of runs of the listed tests executed "
1033                 f"against {item[u'title']}.\n"
1034             )
1035         header.extend(
1036             [
1037                 f"{table[u'reference'][u'title']} "
1038                 f"Avg({table[u'include-tests']})",
1039                 f"{table[u'reference'][u'title']} "
1040                 f"Stdev({table[u'include-tests']})",
1041                 f"{table[u'compare'][u'title']} "
1042                 f"Avg({table[u'include-tests']})",
1043                 f"{table[u'compare'][u'title']} "
1044                 f"Stdev({table[u'include-tests']})",
1045                 f"Diff({table[u'reference'][u'title']},"
1046                 f"{table[u'compare'][u'title']})",
1047                 u"Stdev(Diff)"
1048             ]
1049         )
1050         header_str = u";".join(header) + u"\n"
1051         legend += (
1052             f"{table[u'reference'][u'title']} "
1053             f"Avg({table[u'include-tests']}): "
1054             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
1055             f"series of runs of the listed tests executed against "
1056             f"{table[u'reference'][u'title']}.\n"
1057             f"{table[u'reference'][u'title']} "
1058             f"Stdev({table[u'include-tests']}): "
1059             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1060             f"computed from a series of runs of the listed tests executed "
1061             f"against {table[u'reference'][u'title']}.\n"
1062             f"{table[u'compare'][u'title']} "
1063             f"Avg({table[u'include-tests']}): "
1064             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
1065             f"series of runs of the listed tests executed against "
1066             f"{table[u'compare'][u'title']}.\n"
1067             f"{table[u'compare'][u'title']} "
1068             f"Stdev({table[u'include-tests']}): "
1069             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1070             f"computed from a series of runs of the listed tests executed "
1071             f"against {table[u'compare'][u'title']}.\n"
1072             f"Diff({table[u'reference'][u'title']},"
1073             f"{table[u'compare'][u'title']}): "
1074             f"Percentage change calculated for mean values.\n"
1075             u"Stdev(Diff): "
1076             u"Standard deviation of percentage change calculated for mean "
1077             u"values.\n"
1078             u"NT: Not Tested\n"
1079         )
1080     except (AttributeError, KeyError) as err:
1081         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1082         return
1083
1084     # Prepare data to the table:
1085     tbl_dict = dict()
1086     for job, builds in table[u"reference"][u"data"].items():
1087         for build in builds:
1088             for tst_name, tst_data in data[job][str(build)].items():
1089                 if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
1090                     continue
1091                 tst_name_mod = _tpc_modify_test_name(tst_name)
1092                 if (u"across topologies" in table[u"title"].lower() or
1093                         (u" 3n-" in table[u"title"].lower() and
1094                          u" 2n-" in table[u"title"].lower())):
1095                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1096                 if tbl_dict.get(tst_name_mod, None) is None:
1097                     name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
1098                     if u"across testbeds" in table[u"title"].lower() or \
1099                             u"across topologies" in table[u"title"].lower():
1100                         name = _tpc_modify_displayed_test_name(name)
1101                     tbl_dict[tst_name_mod] = {
1102                         u"name": name,
1103                         u"ref-data": list(),
1104                         u"cmp-data": list()
1105                     }
1106                 _tpc_insert_data(
1107                     target=tbl_dict[tst_name_mod][u"ref-data"],
1108                     src=tst_data,
1109                     include_tests=table[u"include-tests"]
1110                 )
1111
1112     replacement = table[u"reference"].get(u"data-replacement", None)
1113     if replacement:
1114         create_new_list = True
1115         rpl_data = input_data.filter_data(
1116             table, data=replacement, continue_on_error=True)
1117         for job, builds in replacement.items():
1118             for build in builds:
1119                 for tst_name, tst_data in rpl_data[job][str(build)].items():
1120                     if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
1121                         continue
1122                     tst_name_mod = _tpc_modify_test_name(tst_name)
1123                     if (u"across topologies" in table[u"title"].lower() or
1124                             (u" 3n-" in table[u"title"].lower() and
1125                              u" 2n-" in table[u"title"].lower())):
1126                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1127                     if tbl_dict.get(tst_name_mod, None) is None:
1128                         name = \
1129                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
1130                         if u"across testbeds" in table[u"title"].lower() or \
1131                                 u"across topologies" in table[u"title"].lower():
1132                             name = _tpc_modify_displayed_test_name(name)
1133                         tbl_dict[tst_name_mod] = {
1134                             u"name": name,
1135                             u"ref-data": list(),
1136                             u"cmp-data": list()
1137                         }
1138                     if create_new_list:
1139                         create_new_list = False
1140                         tbl_dict[tst_name_mod][u"ref-data"] = list()
1141
1142                     _tpc_insert_data(
1143                         target=tbl_dict[tst_name_mod][u"ref-data"],
1144                         src=tst_data,
1145                         include_tests=table[u"include-tests"]
1146                     )
1147
1148     for job, builds in table[u"compare"][u"data"].items():
1149         for build in builds:
1150             for tst_name, tst_data in data[job][str(build)].items():
1151                 if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
1152                     continue
1153                 tst_name_mod = _tpc_modify_test_name(tst_name)
1154                 if (u"across topologies" in table[u"title"].lower() or
1155                         (u" 3n-" in table[u"title"].lower() and
1156                          u" 2n-" in table[u"title"].lower())):
1157                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1158                 if tbl_dict.get(tst_name_mod, None) is None:
1159                     name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
1160                     if u"across testbeds" in table[u"title"].lower() or \
1161                             u"across topologies" in table[u"title"].lower():
1162                         name = _tpc_modify_displayed_test_name(name)
1163                     tbl_dict[tst_name_mod] = {
1164                         u"name": name,
1165                         u"ref-data": list(),
1166                         u"cmp-data": list()
1167                     }
1168                 _tpc_insert_data(
1169                     target=tbl_dict[tst_name_mod][u"cmp-data"],
1170                     src=tst_data,
1171                     include_tests=table[u"include-tests"]
1172                 )
1173
1174     replacement = table[u"compare"].get(u"data-replacement", None)
1175     if replacement:
1176         create_new_list = True
1177         rpl_data = input_data.filter_data(
1178             table, data=replacement, continue_on_error=True)
1179         for job, builds in replacement.items():
1180             for build in builds:
1181                 for tst_name, tst_data in rpl_data[job][str(build)].items():
1182                     if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
1183                         continue
1184                     tst_name_mod = _tpc_modify_test_name(tst_name)
1185                     if (u"across topologies" in table[u"title"].lower() or
1186                             (u" 3n-" in table[u"title"].lower() and
1187                              u" 2n-" in table[u"title"].lower())):
1188                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1189                     if tbl_dict.get(tst_name_mod, None) is None:
1190                         name = \
1191                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
1192                         if u"across testbeds" in table[u"title"].lower() or \
1193                                 u"across topologies" in table[u"title"].lower():
1194                             name = _tpc_modify_displayed_test_name(name)
1195                         tbl_dict[tst_name_mod] = {
1196                             u"name": name,
1197                             u"ref-data": list(),
1198                             u"cmp-data": list()
1199                         }
1200                     if create_new_list:
1201                         create_new_list = False
1202                         tbl_dict[tst_name_mod][u"cmp-data"] = list()
1203
1204                     _tpc_insert_data(
1205                         target=tbl_dict[tst_name_mod][u"cmp-data"],
1206                         src=tst_data,
1207                         include_tests=table[u"include-tests"]
1208                     )
1209
1210     for item in history:
1211         for job, builds in item[u"data"].items():
1212             for build in builds:
1213                 for tst_name, tst_data in data[job][str(build)].items():
1214                     if item[u"nic"] not in tst_data[u"tags"]:
1215                         continue
1216                     tst_name_mod = _tpc_modify_test_name(tst_name)
1217                     if (u"across topologies" in table[u"title"].lower() or
1218                             (u" 3n-" in table[u"title"].lower() and
1219                              u" 2n-" in table[u"title"].lower())):
1220                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1221                     if tbl_dict.get(tst_name_mod, None) is None:
1222                         continue
1223                     if tbl_dict[tst_name_mod].get(u"history", None) is None:
1224                         tbl_dict[tst_name_mod][u"history"] = OrderedDict()
1225                     if tbl_dict[tst_name_mod][u"history"].\
1226                             get(item[u"title"], None) is None:
1227                         tbl_dict[tst_name_mod][u"history"][item[
1228                             u"title"]] = list()
1229                     try:
1230                         if table[u"include-tests"] == u"MRR":
1231                             res = (tst_data[u"result"][u"receive-rate"],
1232                                    tst_data[u"result"][u"receive-stdev"])
1233                         elif table[u"include-tests"] == u"PDR":
1234                             res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
1235                         elif table[u"include-tests"] == u"NDR":
1236                             res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
1237                         else:
1238                             continue
1239                         tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
1240                             append(res)
1241                     except (TypeError, KeyError):
1242                         pass
1243
1244     tbl_lst = list()
1245     for tst_name in tbl_dict:
1246         item = [tbl_dict[tst_name][u"name"], ]
1247         if history:
1248             if tbl_dict[tst_name].get(u"history", None) is not None:
1249                 for hist_data in tbl_dict[tst_name][u"history"].values():
1250                     if hist_data:
1251                         if table[u"include-tests"] == u"MRR":
1252                             item.append(round(hist_data[0][0] / 1e6, 1))
1253                             item.append(round(hist_data[0][1] / 1e6, 1))
1254                         else:
1255                             item.append(round(mean(hist_data) / 1e6, 1))
1256                             item.append(round(stdev(hist_data) / 1e6, 1))
1257                     else:
1258                         item.extend([u"NT", u"NT"])
1259             else:
1260                 item.extend([u"NT", u"NT"])
1261         data_r = tbl_dict[tst_name][u"ref-data"]
1262         if data_r:
1263             if table[u"include-tests"] == u"MRR":
1264                 data_r_mean = data_r[0][0]
1265                 data_r_stdev = data_r[0][1]
1266             else:
1267                 data_r_mean = mean(data_r)
1268                 data_r_stdev = stdev(data_r)
1269             item.append(round(data_r_mean / 1e6, 1))
1270             item.append(round(data_r_stdev / 1e6, 1))
1271         else:
1272             data_r_mean = None
1273             data_r_stdev = None
1274             item.extend([u"NT", u"NT"])
1275         data_c = tbl_dict[tst_name][u"cmp-data"]
1276         if data_c:
1277             if table[u"include-tests"] == u"MRR":
1278                 data_c_mean = data_c[0][0]
1279                 data_c_stdev = data_c[0][1]
1280             else:
1281                 data_c_mean = mean(data_c)
1282                 data_c_stdev = stdev(data_c)
1283             item.append(round(data_c_mean / 1e6, 1))
1284             item.append(round(data_c_stdev / 1e6, 1))
1285         else:
1286             data_c_mean = None
1287             data_c_stdev = None
1288             item.extend([u"NT", u"NT"])
1289         if item[-2] == u"NT":
1290             pass
1291         elif item[-4] == u"NT":
1292             item.append(u"New in CSIT-2001")
1293             item.append(u"New in CSIT-2001")
1294         elif data_r_mean is not None and data_c_mean is not None:
1295             delta, d_stdev = relative_change_stdev(
1296                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
1297             )
1298             try:
1299                 item.append(round(delta))
1300             except ValueError:
1301                 item.append(delta)
1302             try:
1303                 item.append(round(d_stdev))
1304             except ValueError:
1305                 item.append(d_stdev)
1306         if rca_data:
1307             rca_nr = rca_data.get(item[0], u"-")
1308             item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1309         if (len(item) == len(header)) and (item[-4] != u"NT"):
1310             tbl_lst.append(item)
1311
1312     tbl_lst = _tpc_sort_table(tbl_lst)
1313
1314     # Generate csv tables:
1315     csv_file = f"{table[u'output-file']}.csv"
1316     with open(csv_file, u"wt") as file_handler:
1317         file_handler.write(header_str)
1318         for test in tbl_lst:
1319             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1320
1321     txt_file_name = f"{table[u'output-file']}.txt"
1322     convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
1323
1324     footnote = u""
1325     with open(txt_file_name, u'a') as txt_file:
1326         txt_file.write(legend)
1327         if rca_data:
1328             footnote = rca_data.get(u"footnote", u"")
1329             if footnote:
1330                 txt_file.write(footnote)
1331         txt_file.write(u":END")
1332
1333     # Generate html table:
1334     _tpc_generate_html_table(
1335         header,
1336         tbl_lst,
1337         table[u'output-file'],
1338         legend=legend,
1339         footnote=footnote
1340     )
1341
1342
1343 def table_nics_comparison(table, input_data):
1344     """Generate the table(s) with algorithm: table_nics_comparison
1345     specified in the specification file.
1346
1347     :param table: Table to generate.
1348     :param input_data: Data to process.
1349     :type table: pandas.Series
1350     :type input_data: InputData
1351     """
1352
1353     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1354
1355     # Transform the data
1356     logging.info(
1357         f"    Creating the data set for the {table.get(u'type', u'')} "
1358         f"{table.get(u'title', u'')}."
1359     )
1360     data = input_data.filter_data(table, continue_on_error=True)
1361
1362     # Prepare the header of the tables
1363     try:
1364         header = [
1365             u"Test Case",
1366             f"{table[u'reference'][u'title']} "
1367             f"Avg({table[u'include-tests']})",
1368             f"{table[u'reference'][u'title']} "
1369             f"Stdev({table[u'include-tests']})",
1370             f"{table[u'compare'][u'title']} "
1371             f"Avg({table[u'include-tests']})",
1372             f"{table[u'compare'][u'title']} "
1373             f"Stdev({table[u'include-tests']})",
1374             f"Diff({table[u'reference'][u'title']},"
1375             f"{table[u'compare'][u'title']})",
1376             u"Stdev(Diff)"
1377         ]
1378         legend = (
1379             u"\nLegend:\n"
1380             f"{table[u'reference'][u'title']} "
1381             f"Avg({table[u'include-tests']}): "
1382             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
1383             f"series of runs of the listed tests executed using "
1384             f"{table[u'reference'][u'title']} NIC.\n"
1385             f"{table[u'reference'][u'title']} "
1386             f"Stdev({table[u'include-tests']}): "
1387             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1388             f"computed from a series of runs of the listed tests executed "
1389             f"using {table[u'reference'][u'title']} NIC.\n"
1390             f"{table[u'compare'][u'title']} "
1391             f"Avg({table[u'include-tests']}): "
1392             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
1393             f"series of runs of the listed tests executed using "
1394             f"{table[u'compare'][u'title']} NIC.\n"
1395             f"{table[u'compare'][u'title']} "
1396             f"Stdev({table[u'include-tests']}): "
1397             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1398             f"computed from a series of runs of the listed tests executed "
1399             f"using {table[u'compare'][u'title']} NIC.\n"
1400             f"Diff({table[u'reference'][u'title']},"
1401             f"{table[u'compare'][u'title']}): "
1402             f"Percentage change calculated for mean values.\n"
1403             u"Stdev(Diff): "
1404             u"Standard deviation of percentage change calculated for mean "
1405             u"values.\n"
1406             u":END"
1407         )
1408
1409     except (AttributeError, KeyError) as err:
1410         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1411         return
1412
1413     # Prepare data to the table:
1414     tbl_dict = dict()
1415     for job, builds in table[u"data"].items():
1416         for build in builds:
1417             for tst_name, tst_data in data[job][str(build)].items():
1418                 tst_name_mod = _tpc_modify_test_name(tst_name)
1419                 if tbl_dict.get(tst_name_mod, None) is None:
1420                     name = u"-".join(tst_data[u"name"].split(u"-")[:-1])
1421                     tbl_dict[tst_name_mod] = {
1422                         u"name": name,
1423                         u"ref-data": list(),
1424                         u"cmp-data": list()
1425                     }
1426                 try:
1427                     if table[u"include-tests"] == u"MRR":
1428                         result = (tst_data[u"result"][u"receive-rate"],
1429                                   tst_data[u"result"][u"receive-stdev"])
1430                     elif table[u"include-tests"] == u"PDR":
1431                         result = tst_data[u"throughput"][u"PDR"][u"LOWER"]
1432                     elif table[u"include-tests"] == u"NDR":
1433                         result = tst_data[u"throughput"][u"NDR"][u"LOWER"]
1434                     else:
1435                         continue
1436
1437                     if result and \
1438                             table[u"reference"][u"nic"] in tst_data[u"tags"]:
1439                         tbl_dict[tst_name_mod][u"ref-data"].append(result)
1440                     elif result and \
1441                             table[u"compare"][u"nic"] in tst_data[u"tags"]:
1442                         tbl_dict[tst_name_mod][u"cmp-data"].append(result)
1443                 except (TypeError, KeyError) as err:
1444                     logging.debug(f"No data for {tst_name}\n{repr(err)}")
1445                     # No data in output.xml for this test
1446
1447     tbl_lst = list()
1448     for tst_name in tbl_dict:
1449         item = [tbl_dict[tst_name][u"name"], ]
1450         data_r = tbl_dict[tst_name][u"ref-data"]
1451         if data_r:
1452             if table[u"include-tests"] == u"MRR":
1453                 data_r_mean = data_r[0][0]
1454                 data_r_stdev = data_r[0][1]
1455             else:
1456                 data_r_mean = mean(data_r)
1457                 data_r_stdev = stdev(data_r)
1458             item.append(round(data_r_mean / 1e6, 1))
1459             item.append(round(data_r_stdev / 1e6, 1))
1460         else:
1461             data_r_mean = None
1462             data_r_stdev = None
1463             item.extend([None, None])
1464         data_c = tbl_dict[tst_name][u"cmp-data"]
1465         if data_c:
1466             if table[u"include-tests"] == u"MRR":
1467                 data_c_mean = data_c[0][0]
1468                 data_c_stdev = data_c[0][1]
1469             else:
1470                 data_c_mean = mean(data_c)
1471                 data_c_stdev = stdev(data_c)
1472             item.append(round(data_c_mean / 1e6, 1))
1473             item.append(round(data_c_stdev / 1e6, 1))
1474         else:
1475             data_c_mean = None
1476             data_c_stdev = None
1477             item.extend([None, None])
1478         if data_r_mean is not None and data_c_mean is not None:
1479             delta, d_stdev = relative_change_stdev(
1480                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
1481             )
1482             try:
1483                 item.append(round(delta))
1484             except ValueError:
1485                 item.append(delta)
1486             try:
1487                 item.append(round(d_stdev))
1488             except ValueError:
1489                 item.append(d_stdev)
1490             tbl_lst.append(item)
1491
1492     # Sort the table according to the relative change
1493     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1494
1495     # Generate csv tables:
1496     with open(f"{table[u'output-file']}.csv", u"wt") as file_handler:
1497         file_handler.write(u";".join(header) + u"\n")
1498         for test in tbl_lst:
1499             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1500
1501     convert_csv_to_pretty_txt(f"{table[u'output-file']}.csv",
1502                               f"{table[u'output-file']}.txt",
1503                               delimiter=u";")
1504
1505     with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
1506         txt_file.write(legend)
1507
1508     # Generate html table:
1509     _tpc_generate_html_table(
1510         header,
1511         tbl_lst,
1512         table[u'output-file'],
1513         legend=legend
1514     )
1515
1516
1517 def table_soak_vs_ndr(table, input_data):
1518     """Generate the table(s) with algorithm: table_soak_vs_ndr
1519     specified in the specification file.
1520
1521     :param table: Table to generate.
1522     :param input_data: Data to process.
1523     :type table: pandas.Series
1524     :type input_data: InputData
1525     """
1526
1527     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1528
1529     # Transform the data
1530     logging.info(
1531         f"    Creating the data set for the {table.get(u'type', u'')} "
1532         f"{table.get(u'title', u'')}."
1533     )
1534     data = input_data.filter_data(table, continue_on_error=True)
1535
1536     # Prepare the header of the table
1537     try:
1538         header = [
1539             u"Test Case",
1540             f"Avg({table[u'reference'][u'title']})",
1541             f"Stdev({table[u'reference'][u'title']})",
1542             f"Avg({table[u'compare'][u'title']})",
1543             f"Stdev{table[u'compare'][u'title']})",
1544             u"Diff",
1545             u"Stdev(Diff)"
1546         ]
1547         header_str = u";".join(header) + u"\n"
1548         legend = (
1549             u"\nLegend:\n"
1550             f"Avg({table[u'reference'][u'title']}): "
1551             f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
1552             f"from a series of runs of the listed tests.\n"
1553             f"Stdev({table[u'reference'][u'title']}): "
1554             f"Standard deviation value of {table[u'reference'][u'title']} "
1555             f"[Mpps] computed from a series of runs of the listed tests.\n"
1556             f"Avg({table[u'compare'][u'title']}): "
1557             f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
1558             f"a series of runs of the listed tests.\n"
1559             f"Stdev({table[u'compare'][u'title']}): "
1560             f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
1561             f"computed from a series of runs of the listed tests.\n"
1562             f"Diff({table[u'reference'][u'title']},"
1563             f"{table[u'compare'][u'title']}): "
1564             f"Percentage change calculated for mean values.\n"
1565             u"Stdev(Diff): "
1566             u"Standard deviation of percentage change calculated for mean "
1567             u"values.\n"
1568             u":END"
1569         )
1570     except (AttributeError, KeyError) as err:
1571         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1572         return
1573
1574     # Create a list of available SOAK test results:
1575     tbl_dict = dict()
1576     for job, builds in table[u"compare"][u"data"].items():
1577         for build in builds:
1578             for tst_name, tst_data in data[job][str(build)].items():
1579                 if tst_data[u"type"] == u"SOAK":
1580                     tst_name_mod = tst_name.replace(u"-soak", u"")
1581                     if tbl_dict.get(tst_name_mod, None) is None:
1582                         groups = re.search(REGEX_NIC, tst_data[u"parent"])
1583                         nic = groups.group(0) if groups else u""
1584                         name = (
1585                             f"{nic}-"
1586                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
1587                         )
1588                         tbl_dict[tst_name_mod] = {
1589                             u"name": name,
1590                             u"ref-data": list(),
1591                             u"cmp-data": list()
1592                         }
1593                     try:
1594                         tbl_dict[tst_name_mod][u"cmp-data"].append(
1595                             tst_data[u"throughput"][u"LOWER"])
1596                     except (KeyError, TypeError):
1597                         pass
1598     tests_lst = tbl_dict.keys()
1599
1600     # Add corresponding NDR test results:
1601     for job, builds in table[u"reference"][u"data"].items():
1602         for build in builds:
1603             for tst_name, tst_data in data[job][str(build)].items():
1604                 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
1605                     replace(u"-mrr", u"")
1606                 if tst_name_mod not in tests_lst:
1607                     continue
1608                 try:
1609                     if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
1610                         continue
1611                     if table[u"include-tests"] == u"MRR":
1612                         result = (tst_data[u"result"][u"receive-rate"],
1613                                   tst_data[u"result"][u"receive-stdev"])
1614                     elif table[u"include-tests"] == u"PDR":
1615                         result = \
1616                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
1617                     elif table[u"include-tests"] == u"NDR":
1618                         result = \
1619                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
1620                     else:
1621                         result = None
1622                     if result is not None:
1623                         tbl_dict[tst_name_mod][u"ref-data"].append(
1624                             result)
1625                 except (KeyError, TypeError):
1626                     continue
1627
1628     tbl_lst = list()
1629     for tst_name in tbl_dict:
1630         item = [tbl_dict[tst_name][u"name"], ]
1631         data_r = tbl_dict[tst_name][u"ref-data"]
1632         if data_r:
1633             if table[u"include-tests"] == u"MRR":
1634                 data_r_mean = data_r[0][0]
1635                 data_r_stdev = data_r[0][1]
1636             else:
1637                 data_r_mean = mean(data_r)
1638                 data_r_stdev = stdev(data_r)
1639             item.append(round(data_r_mean / 1e6, 1))
1640             item.append(round(data_r_stdev / 1e6, 1))
1641         else:
1642             data_r_mean = None
1643             data_r_stdev = None
1644             item.extend([None, None])
1645         data_c = tbl_dict[tst_name][u"cmp-data"]
1646         if data_c:
1647             if table[u"include-tests"] == u"MRR":
1648                 data_c_mean = data_c[0][0]
1649                 data_c_stdev = data_c[0][1]
1650             else:
1651                 data_c_mean = mean(data_c)
1652                 data_c_stdev = stdev(data_c)
1653             item.append(round(data_c_mean / 1e6, 1))
1654             item.append(round(data_c_stdev / 1e6, 1))
1655         else:
1656             data_c_mean = None
1657             data_c_stdev = None
1658             item.extend([None, None])
1659         if data_r_mean is not None and data_c_mean is not None:
1660             delta, d_stdev = relative_change_stdev(
1661                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
1662             try:
1663                 item.append(round(delta))
1664             except ValueError:
1665                 item.append(delta)
1666             try:
1667                 item.append(round(d_stdev))
1668             except ValueError:
1669                 item.append(d_stdev)
1670             tbl_lst.append(item)
1671
1672     # Sort the table according to the relative change
1673     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1674
1675     # Generate csv tables:
1676     csv_file = f"{table[u'output-file']}.csv"
1677     with open(csv_file, u"wt") as file_handler:
1678         file_handler.write(header_str)
1679         for test in tbl_lst:
1680             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1681
1682     convert_csv_to_pretty_txt(
1683         csv_file, f"{table[u'output-file']}.txt", delimiter=u";"
1684     )
1685     with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
1686         txt_file.write(legend)
1687
1688     # Generate html table:
1689     _tpc_generate_html_table(
1690         header,
1691         tbl_lst,
1692         table[u'output-file'],
1693         legend=legend
1694     )
1695
1696
1697 def table_perf_trending_dash(table, input_data):
1698     """Generate the table(s) with algorithm:
1699     table_perf_trending_dash
1700     specified in the specification file.
1701
1702     :param table: Table to generate.
1703     :param input_data: Data to process.
1704     :type table: pandas.Series
1705     :type input_data: InputData
1706     """
1707
1708     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1709
1710     # Transform the data
1711     logging.info(
1712         f"    Creating the data set for the {table.get(u'type', u'')} "
1713         f"{table.get(u'title', u'')}."
1714     )
1715     data = input_data.filter_data(table, continue_on_error=True)
1716
1717     # Prepare the header of the tables
1718     header = [
1719         u"Test Case",
1720         u"Trend [Mpps]",
1721         u"Short-Term Change [%]",
1722         u"Long-Term Change [%]",
1723         u"Regressions [#]",
1724         u"Progressions [#]"
1725     ]
1726     header_str = u",".join(header) + u"\n"
1727
1728     # Prepare data to the table:
1729     tbl_dict = dict()
1730     for job, builds in table[u"data"].items():
1731         for build in builds:
1732             for tst_name, tst_data in data[job][str(build)].items():
1733                 if tst_name.lower() in table.get(u"ignore-list", list()):
1734                     continue
1735                 if tbl_dict.get(tst_name, None) is None:
1736                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1737                     if not groups:
1738                         continue
1739                     nic = groups.group(0)
1740                     tbl_dict[tst_name] = {
1741                         u"name": f"{nic}-{tst_data[u'name']}",
1742                         u"data": OrderedDict()
1743                     }
1744                 try:
1745                     tbl_dict[tst_name][u"data"][str(build)] = \
1746                         tst_data[u"result"][u"receive-rate"]
1747                 except (TypeError, KeyError):
1748                     pass  # No data in output.xml for this test
1749
1750     tbl_lst = list()
1751     for tst_name in tbl_dict:
1752         data_t = tbl_dict[tst_name][u"data"]
1753         if len(data_t) < 2:
1754             continue
1755
1756         classification_lst, avgs = classify_anomalies(data_t)
1757
1758         win_size = min(len(data_t), table[u"window"])
1759         long_win_size = min(len(data_t), table[u"long-trend-window"])
1760
1761         try:
1762             max_long_avg = max(
1763                 [x for x in avgs[-long_win_size:-win_size]
1764                  if not isnan(x)])
1765         except ValueError:
1766             max_long_avg = nan
1767         last_avg = avgs[-1]
1768         avg_week_ago = avgs[max(-win_size, -len(avgs))]
1769
1770         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
1771             rel_change_last = nan
1772         else:
1773             rel_change_last = round(
1774                 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
1775
1776         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
1777             rel_change_long = nan
1778         else:
1779             rel_change_long = round(
1780                 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
1781
1782         if classification_lst:
1783             if isnan(rel_change_last) and isnan(rel_change_long):
1784                 continue
1785             if isnan(last_avg) or isnan(rel_change_last) or \
1786                     isnan(rel_change_long):
1787                 continue
1788             tbl_lst.append(
1789                 [tbl_dict[tst_name][u"name"],
1790                  round(last_avg / 1e6, 2),
1791                  rel_change_last,
1792                  rel_change_long,
1793                  classification_lst[-win_size:].count(u"regression"),
1794                  classification_lst[-win_size:].count(u"progression")])
1795
1796     tbl_lst.sort(key=lambda rel: rel[0])
1797
1798     tbl_sorted = list()
1799     for nrr in range(table[u"window"], -1, -1):
1800         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
1801         for nrp in range(table[u"window"], -1, -1):
1802             tbl_out = [item for item in tbl_reg if item[5] == nrp]
1803             tbl_out.sort(key=lambda rel: rel[2])
1804             tbl_sorted.extend(tbl_out)
1805
1806     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1807
1808     logging.info(f"    Writing file: {file_name}")
1809     with open(file_name, u"wt") as file_handler:
1810         file_handler.write(header_str)
1811         for test in tbl_sorted:
1812             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1813
1814     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1815     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1816
1817
1818 def _generate_url(testbed, test_name):
1819     """Generate URL to a trending plot from the name of the test case.
1820
1821     :param testbed: The testbed used for testing.
1822     :param test_name: The name of the test case.
1823     :type testbed: str
1824     :type test_name: str
1825     :returns: The URL to the plot with the trending data for the given test
1826         case.
1827     :rtype str
1828     """
1829
1830     if u"x520" in test_name:
1831         nic = u"x520"
1832     elif u"x710" in test_name:
1833         nic = u"x710"
1834     elif u"xl710" in test_name:
1835         nic = u"xl710"
1836     elif u"xxv710" in test_name:
1837         nic = u"xxv710"
1838     elif u"vic1227" in test_name:
1839         nic = u"vic1227"
1840     elif u"vic1385" in test_name:
1841         nic = u"vic1385"
1842     elif u"x553" in test_name:
1843         nic = u"x553"
1844     elif u"cx556" in test_name or u"cx556a" in test_name:
1845         nic = u"cx556a"
1846     else:
1847         nic = u""
1848
1849     if u"64b" in test_name:
1850         frame_size = u"64b"
1851     elif u"78b" in test_name:
1852         frame_size = u"78b"
1853     elif u"imix" in test_name:
1854         frame_size = u"imix"
1855     elif u"9000b" in test_name:
1856         frame_size = u"9000b"
1857     elif u"1518b" in test_name:
1858         frame_size = u"1518b"
1859     elif u"114b" in test_name:
1860         frame_size = u"114b"
1861     else:
1862         frame_size = u""
1863
1864     if u"1t1c" in test_name or \
1865         (u"-1c-" in test_name and
1866          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1867         cores = u"1t1c"
1868     elif u"2t2c" in test_name or \
1869          (u"-2c-" in test_name and
1870           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1871         cores = u"2t2c"
1872     elif u"4t4c" in test_name or \
1873          (u"-4c-" in test_name and
1874           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1875         cores = u"4t4c"
1876     elif u"2t1c" in test_name or \
1877          (u"-1c-" in test_name and
1878           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1879         cores = u"2t1c"
1880     elif u"4t2c" in test_name or \
1881          (u"-2c-" in test_name and
1882           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1883         cores = u"4t2c"
1884     elif u"8t4c" in test_name or \
1885          (u"-4c-" in test_name and
1886           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1887         cores = u"8t4c"
1888     else:
1889         cores = u""
1890
1891     if u"testpmd" in test_name:
1892         driver = u"testpmd"
1893     elif u"l3fwd" in test_name:
1894         driver = u"l3fwd"
1895     elif u"avf" in test_name:
1896         driver = u"avf"
1897     elif u"rdma" in test_name:
1898         driver = u"rdma"
1899     elif u"dnv" in testbed or u"tsh" in testbed:
1900         driver = u"ixgbe"
1901     else:
1902         driver = u"dpdk"
1903
1904     if u"acl" in test_name or \
1905             u"macip" in test_name or \
1906             u"nat" in test_name or \
1907             u"policer" in test_name or \
1908             u"cop" in test_name:
1909         bsf = u"features"
1910     elif u"scale" in test_name:
1911         bsf = u"scale"
1912     elif u"base" in test_name:
1913         bsf = u"base"
1914     else:
1915         bsf = u"base"
1916
1917     if u"114b" in test_name and u"vhost" in test_name:
1918         domain = u"vts"
1919     elif u"testpmd" in test_name or u"l3fwd" in test_name:
1920         domain = u"dpdk"
1921     elif u"memif" in test_name:
1922         domain = u"container_memif"
1923     elif u"srv6" in test_name:
1924         domain = u"srv6"
1925     elif u"vhost" in test_name:
1926         domain = u"vhost"
1927         if u"vppl2xc" in test_name:
1928             driver += u"-vpp"
1929         else:
1930             driver += u"-testpmd"
1931         if u"lbvpplacp" in test_name:
1932             bsf += u"-link-bonding"
1933     elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1934         domain = u"nf_service_density_vnfc"
1935     elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1936         domain = u"nf_service_density_cnfc"
1937     elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1938         domain = u"nf_service_density_cnfp"
1939     elif u"ipsec" in test_name:
1940         domain = u"ipsec"
1941         if u"sw" in test_name:
1942             bsf += u"-sw"
1943         elif u"hw" in test_name:
1944             bsf += u"-hw"
1945     elif u"ethip4vxlan" in test_name:
1946         domain = u"ip4_tunnels"
1947     elif u"ip4base" in test_name or u"ip4scale" in test_name:
1948         domain = u"ip4"
1949     elif u"ip6base" in test_name or u"ip6scale" in test_name:
1950         domain = u"ip6"
1951     elif u"l2xcbase" in test_name or \
1952             u"l2xcscale" in test_name or \
1953             u"l2bdbasemaclrn" in test_name or \
1954             u"l2bdscale" in test_name or \
1955             u"l2patch" in test_name:
1956         domain = u"l2"
1957     else:
1958         domain = u""
1959
1960     file_name = u"-".join((domain, testbed, nic)) + u".html#"
1961     anchor_name = u"-".join((frame_size, cores, bsf, driver))
1962
1963     return file_name + anchor_name
1964
1965
1966 def table_perf_trending_dash_html(table, input_data):
1967     """Generate the table(s) with algorithm:
1968     table_perf_trending_dash_html specified in the specification
1969     file.
1970
1971     :param table: Table to generate.
1972     :param input_data: Data to process.
1973     :type table: dict
1974     :type input_data: InputData
1975     """
1976
1977     _ = input_data
1978
1979     if not table.get(u"testbed", None):
1980         logging.error(
1981             f"The testbed is not defined for the table "
1982             f"{table.get(u'title', u'')}."
1983         )
1984         return
1985
1986     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1987
1988     try:
1989         with open(table[u"input-file"], u'rt') as csv_file:
1990             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1991     except KeyError:
1992         logging.warning(u"The input file is not defined.")
1993         return
1994     except csv.Error as err:
1995         logging.warning(
1996             f"Not possible to process the file {table[u'input-file']}.\n"
1997             f"{repr(err)}"
1998         )
1999         return
2000
2001     # Table:
2002     dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
2003
2004     # Table header:
2005     trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
2006     for idx, item in enumerate(csv_lst[0]):
2007         alignment = u"left" if idx == 0 else u"center"
2008         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
2009         thead.text = item
2010
2011     # Rows:
2012     colors = {
2013         u"regression": (
2014             u"#ffcccc",
2015             u"#ff9999"
2016         ),
2017         u"progression": (
2018             u"#c6ecc6",
2019             u"#9fdf9f"
2020         ),
2021         u"normal": (
2022             u"#e9f1fb",
2023             u"#d4e4f7"
2024         )
2025     }
2026     for r_idx, row in enumerate(csv_lst[1:]):
2027         if int(row[4]):
2028             color = u"regression"
2029         elif int(row[5]):
2030             color = u"progression"
2031         else:
2032             color = u"normal"
2033         trow = ET.SubElement(
2034             dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
2035         )
2036
2037         # Columns:
2038         for c_idx, item in enumerate(row):
2039             tdata = ET.SubElement(
2040                 trow,
2041                 u"td",
2042                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
2043             )
2044             # Name:
2045             if c_idx == 0:
2046                 ref = ET.SubElement(
2047                     tdata,
2048                     u"a",
2049                     attrib=dict(
2050                         href=f"../trending/"
2051                              f"{_generate_url(table.get(u'testbed', ''), item)}"
2052                     )
2053                 )
2054                 ref.text = item
2055             else:
2056                 tdata.text = item
2057     try:
2058         with open(table[u"output-file"], u'w') as html_file:
2059             logging.info(f"    Writing file: {table[u'output-file']}")
2060             html_file.write(u".. raw:: html\n\n\t")
2061             html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
2062             html_file.write(u"\n\t<p><br><br></p>\n")
2063     except KeyError:
2064         logging.warning(u"The output file is not defined.")
2065         return
2066
2067
2068 def table_last_failed_tests(table, input_data):
2069     """Generate the table(s) with algorithm: table_last_failed_tests
2070     specified in the specification file.
2071
2072     :param table: Table to generate.
2073     :param input_data: Data to process.
2074     :type table: pandas.Series
2075     :type input_data: InputData
2076     """
2077
2078     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
2079
2080     # Transform the data
2081     logging.info(
2082         f"    Creating the data set for the {table.get(u'type', u'')} "
2083         f"{table.get(u'title', u'')}."
2084     )
2085
2086     data = input_data.filter_data(table, continue_on_error=True)
2087
2088     if data is None or data.empty:
2089         logging.warning(
2090             f"    No data for the {table.get(u'type', u'')} "
2091             f"{table.get(u'title', u'')}."
2092         )
2093         return
2094
2095     tbl_list = list()
2096     for job, builds in table[u"data"].items():
2097         for build in builds:
2098             build = str(build)
2099             try:
2100                 version = input_data.metadata(job, build).get(u"version", u"")
2101             except KeyError:
2102                 logging.error(f"Data for {job}: {build} is not present.")
2103                 return
2104             tbl_list.append(build)
2105             tbl_list.append(version)
2106             failed_tests = list()
2107             passed = 0
2108             failed = 0
2109             for tst_data in data[job][build].values:
2110                 if tst_data[u"status"] != u"FAIL":
2111                     passed += 1
2112                     continue
2113                 failed += 1
2114                 groups = re.search(REGEX_NIC, tst_data[u"parent"])
2115                 if not groups:
2116                     continue
2117                 nic = groups.group(0)
2118                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
2119             tbl_list.append(str(passed))
2120             tbl_list.append(str(failed))
2121             tbl_list.extend(failed_tests)
2122
2123     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
2124     logging.info(f"    Writing file: {file_name}")
2125     with open(file_name, u"wt") as file_handler:
2126         for test in tbl_list:
2127             file_handler.write(test + u'\n')
2128
2129
2130 def table_failed_tests(table, input_data):
2131     """Generate the table(s) with algorithm: table_failed_tests
2132     specified in the specification file.
2133
2134     :param table: Table to generate.
2135     :param input_data: Data to process.
2136     :type table: pandas.Series
2137     :type input_data: InputData
2138     """
2139
2140     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
2141
2142     # Transform the data
2143     logging.info(
2144         f"    Creating the data set for the {table.get(u'type', u'')} "
2145         f"{table.get(u'title', u'')}."
2146     )
2147     data = input_data.filter_data(table, continue_on_error=True)
2148
2149     # Prepare the header of the tables
2150     header = [
2151         u"Test Case",
2152         u"Failures [#]",
2153         u"Last Failure [Time]",
2154         u"Last Failure [VPP-Build-Id]",
2155         u"Last Failure [CSIT-Job-Build-Id]"
2156     ]
2157
2158     # Generate the data for the table according to the model in the table
2159     # specification
2160
2161     now = dt.utcnow()
2162     timeperiod = timedelta(int(table.get(u"window", 7)))
2163
2164     tbl_dict = dict()
2165     for job, builds in table[u"data"].items():
2166         for build in builds:
2167             build = str(build)
2168             for tst_name, tst_data in data[job][build].items():
2169                 if tst_name.lower() in table.get(u"ignore-list", list()):
2170                     continue
2171                 if tbl_dict.get(tst_name, None) is None:
2172                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
2173                     if not groups:
2174                         continue
2175                     nic = groups.group(0)
2176                     tbl_dict[tst_name] = {
2177                         u"name": f"{nic}-{tst_data[u'name']}",
2178                         u"data": OrderedDict()
2179                     }
2180                 try:
2181                     generated = input_data.metadata(job, build).\
2182                         get(u"generated", u"")
2183                     if not generated:
2184                         continue
2185                     then = dt.strptime(generated, u"%Y%m%d %H:%M")
2186                     if (now - then) <= timeperiod:
2187                         tbl_dict[tst_name][u"data"][build] = (
2188                             tst_data[u"status"],
2189                             generated,
2190                             input_data.metadata(job, build).get(u"version",
2191                                                                 u""),
2192                             build
2193                         )
2194                 except (TypeError, KeyError) as err:
2195                     logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
2196
2197     max_fails = 0
2198     tbl_lst = list()
2199     for tst_data in tbl_dict.values():
2200         fails_nr = 0
2201         fails_last_date = u""
2202         fails_last_vpp = u""
2203         fails_last_csit = u""
2204         for val in tst_data[u"data"].values():
2205             if val[0] == u"FAIL":
2206                 fails_nr += 1
2207                 fails_last_date = val[1]
2208                 fails_last_vpp = val[2]
2209                 fails_last_csit = val[3]
2210         if fails_nr:
2211             max_fails = fails_nr if fails_nr > max_fails else max_fails
2212             tbl_lst.append(
2213                 [
2214                     tst_data[u"name"],
2215                     fails_nr,
2216                     fails_last_date,
2217                     fails_last_vpp,
2218                     f"mrr-daily-build-{fails_last_csit}"
2219                 ]
2220             )
2221
2222     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
2223     tbl_sorted = list()
2224     for nrf in range(max_fails, -1, -1):
2225         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
2226         tbl_sorted.extend(tbl_fails)
2227
2228     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
2229     logging.info(f"    Writing file: {file_name}")
2230     with open(file_name, u"wt") as file_handler:
2231         file_handler.write(u",".join(header) + u"\n")
2232         for test in tbl_sorted:
2233             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
2234
2235     logging.info(f"    Writing file: {table[u'output-file']}.txt")
2236     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
2237
2238
2239 def table_failed_tests_html(table, input_data):
2240     """Generate the table(s) with algorithm: table_failed_tests_html
2241     specified in the specification file.
2242
2243     :param table: Table to generate.
2244     :param input_data: Data to process.
2245     :type table: pandas.Series
2246     :type input_data: InputData
2247     """
2248
2249     _ = input_data
2250
2251     if not table.get(u"testbed", None):
2252         logging.error(
2253             f"The testbed is not defined for the table "
2254             f"{table.get(u'title', u'')}."
2255         )
2256         return
2257
2258     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
2259
2260     try:
2261         with open(table[u"input-file"], u'rt') as csv_file:
2262             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
2263     except KeyError:
2264         logging.warning(u"The input file is not defined.")
2265         return
2266     except csv.Error as err:
2267         logging.warning(
2268             f"Not possible to process the file {table[u'input-file']}.\n"
2269             f"{repr(err)}"
2270         )
2271         return
2272
2273     # Table:
2274     failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
2275
2276     # Table header:
2277     trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
2278     for idx, item in enumerate(csv_lst[0]):
2279         alignment = u"left" if idx == 0 else u"center"
2280         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
2281         thead.text = item
2282
2283     # Rows:
2284     colors = (u"#e9f1fb", u"#d4e4f7")
2285     for r_idx, row in enumerate(csv_lst[1:]):
2286         background = colors[r_idx % 2]
2287         trow = ET.SubElement(
2288             failed_tests, u"tr", attrib=dict(bgcolor=background)
2289         )
2290
2291         # Columns:
2292         for c_idx, item in enumerate(row):
2293             tdata = ET.SubElement(
2294                 trow,
2295                 u"td",
2296                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
2297             )
2298             # Name:
2299             if c_idx == 0:
2300                 ref = ET.SubElement(
2301                     tdata,
2302                     u"a",
2303                     attrib=dict(
2304                         href=f"../trending/"
2305                              f"{_generate_url(table.get(u'testbed', ''), item)}"
2306                     )
2307                 )
2308                 ref.text = item
2309             else:
2310                 tdata.text = item
2311     try:
2312         with open(table[u"output-file"], u'w') as html_file:
2313             logging.info(f"    Writing file: {table[u'output-file']}")
2314             html_file.write(u".. raw:: html\n\n\t")
2315             html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
2316             html_file.write(u"\n\t<p><br><br></p>\n")
2317     except KeyError:
2318         logging.warning(u"The output file is not defined.")
2319         return