resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from collections import OrderedDict
  23 from xml.etree import ElementTree as ET
  24 from datetime import datetime as dt
  25 from datetime import timedelta
  26 from copy import deepcopy
  27
  28 import plotly.graph_objects as go
  29 import plotly.offline as ploff
  30 import pandas as pd
  31
  32 from numpy import nan, isnan
  33 from yaml import load, FullLoader, YAMLError
  34
  35 from pal_utils import mean, stdev, classify_anomalies, \
  36     convert_csv_to_pretty_txt, relative_change_stdev
  37
  38
  39 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
  40
  41
  42 def generate_tables(spec, data):
  43     """Generate all tables specified in the specification file.
  44
  45     :param spec: Specification read from the specification file.
  46     :param data: Data to process.
  47     :type spec: Specification
  48     :type data: InputData
  49     """
  50
  51     generator = {
  52         u"table_merged_details": table_merged_details,
  53         u"table_perf_comparison": table_perf_comparison,
  54         u"table_perf_comparison_nic": table_perf_comparison_nic,
  55         u"table_nics_comparison": table_nics_comparison,
  56         u"table_soak_vs_ndr": table_soak_vs_ndr,
  57         u"table_perf_trending_dash": table_perf_trending_dash,
  58         u"table_perf_trending_dash_html": table_perf_trending_dash_html,
  59         u"table_last_failed_tests": table_last_failed_tests,
  60         u"table_failed_tests": table_failed_tests,
  61         u"table_failed_tests_html": table_failed_tests_html,
  62         u"table_oper_data_html": table_oper_data_html,
  63         u"table_comparison": table_comparison
  64     }
  65
  66     logging.info(u"Generating the tables ...")
  67     for table in spec.tables:
  68         try:
  69             generator[table[u"algorithm"]](table, data)
  70         except NameError as err:
  71             logging.error(
  72                 f"Probably algorithm {table[u'algorithm']} is not defined: "
  73                 f"{repr(err)}"
  74             )
  75     logging.info(u"Done.")
  76
  77
  78 def table_oper_data_html(table, input_data):
  79     """Generate the table(s) with algorithm: html_table_oper_data
  80     specified in the specification file.
  81
  82     :param table: Table to generate.
  83     :param input_data: Data to process.
  84     :type table: pandas.Series
  85     :type input_data: InputData
  86     """
  87
  88     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
  89     # Transform the data
  90     logging.info(
  91         f"    Creating the data set for the {table.get(u'type', u'')} "
  92         f"{table.get(u'title', u'')}."
  93     )
  94     data = input_data.filter_data(
  95         table,
  96         params=[u"name", u"parent", u"show-run", u"type"],
  97         continue_on_error=True
  98     )
  99     if data.empty:
 100         return
 101     data = input_data.merge_data(data)
 102
 103     sort_tests = table.get(u"sort", None)
 104     if sort_tests:
 105         args = dict(
 106             inplace=True,
 107             ascending=(sort_tests == u"ascending")
 108         )
 109         data.sort_index(**args)
 110
 111     suites = input_data.filter_data(
 112         table,
 113         continue_on_error=True,
 114         data_set=u"suites"
 115     )
 116     if suites.empty:
 117         return
 118     suites = input_data.merge_data(suites)
 119
 120     def _generate_html_table(tst_data):
 121         """Generate an HTML table with operational data for the given test.
 122
 123         :param tst_data: Test data to be used to generate the table.
 124         :type tst_data: pandas.Series
 125         :returns: HTML table with operational data.
 126         :rtype: str
 127         """
 128
 129         colors = {
 130             u"header": u"#7eade7",
 131             u"empty": u"#ffffff",
 132             u"body": (u"#e9f1fb", u"#d4e4f7")
 133         }
 134
 135         tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
 136
 137         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
 138         thead = ET.SubElement(
 139             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 140         )
 141         thead.text = tst_data[u"name"]
 142
 143         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 144         thead = ET.SubElement(
 145             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 146         )
 147         thead.text = u"\t"
 148
 149         if tst_data.get(u"show-run", u"No Data") == u"No Data":
 150             trow = ET.SubElement(
 151                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 152             )
 153             tcol = ET.SubElement(
 154                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 155             )
 156             tcol.text = u"No Data"
 157
 158             trow = ET.SubElement(
 159                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 160             )
 161             thead = ET.SubElement(
 162                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 163             )
 164             font = ET.SubElement(
 165                 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 166             )
 167             font.text = u"."
 168             return str(ET.tostring(tbl, encoding=u"unicode"))
 169
 170         tbl_hdr = (
 171             u"Name",
 172             u"Nr of Vectors",
 173             u"Nr of Packets",
 174             u"Suspends",
 175             u"Cycles per Packet",
 176             u"Average Vector Size"
 177         )
 178
 179         for dut_data in tst_data[u"show-run"].values():
 180             trow = ET.SubElement(
 181                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 182             )
 183             tcol = ET.SubElement(
 184                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 185             )
 186             if dut_data.get(u"threads", None) is None:
 187                 tcol.text = u"No Data"
 188                 continue
 189
 190             bold = ET.SubElement(tcol, u"b")
 191             bold.text = (
 192                 f"Host IP: {dut_data.get(u'host', '')}, "
 193                 f"Socket: {dut_data.get(u'socket', '')}"
 194             )
 195             trow = ET.SubElement(
 196                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 197             )
 198             thead = ET.SubElement(
 199                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 200             )
 201             thead.text = u"\t"
 202
 203             for thread_nr, thread in dut_data[u"threads"].items():
 204                 trow = ET.SubElement(
 205                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 206                 )
 207                 tcol = ET.SubElement(
 208                     trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 209                 )
 210                 bold = ET.SubElement(tcol, u"b")
 211                 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
 212                 trow = ET.SubElement(
 213                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 214                 )
 215                 for idx, col in enumerate(tbl_hdr):
 216                     tcol = ET.SubElement(
 217                         trow, u"td",
 218                         attrib=dict(align=u"right" if idx else u"left")
 219                     )
 220                     font = ET.SubElement(
 221                         tcol, u"font", attrib=dict(size=u"2")
 222                     )
 223                     bold = ET.SubElement(font, u"b")
 224                     bold.text = col
 225                 for row_nr, row in enumerate(thread):
 226                     trow = ET.SubElement(
 227                         tbl, u"tr",
 228                         attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
 229                     )
 230                     for idx, col in enumerate(row):
 231                         tcol = ET.SubElement(
 232                             trow, u"td",
 233                             attrib=dict(align=u"right" if idx else u"left")
 234                         )
 235                         font = ET.SubElement(
 236                             tcol, u"font", attrib=dict(size=u"2")
 237                         )
 238                         if isinstance(col, float):
 239                             font.text = f"{col:.2f}"
 240                         else:
 241                             font.text = str(col)
 242                 trow = ET.SubElement(
 243                     tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 244                 )
 245                 thead = ET.SubElement(
 246                     trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 247                 )
 248                 thead.text = u"\t"
 249
 250         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 251         thead = ET.SubElement(
 252             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 253         )
 254         font = ET.SubElement(
 255             thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 256         )
 257         font.text = u"."
 258
 259         return str(ET.tostring(tbl, encoding=u"unicode"))
 260
 261     for suite in suites.values:
 262         html_table = str()
 263         for test_data in data.values:
 264             if test_data[u"parent"] not in suite[u"name"]:
 265                 continue
 266             html_table += _generate_html_table(test_data)
 267         if not html_table:
 268             continue
 269         try:
 270             file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
 271             with open(f"{file_name}", u'w') as html_file:
 272                 logging.info(f"    Writing file: {file_name}")
 273                 html_file.write(u".. raw:: html\n\n\t")
 274                 html_file.write(html_table)
 275                 html_file.write(u"\n\t<p><br><br></p>\n")
 276         except KeyError:
 277             logging.warning(u"The output file is not defined.")
 278             return
 279     logging.info(u"  Done.")
 280
 281
 282 def table_merged_details(table, input_data):
 283     """Generate the table(s) with algorithm: table_merged_details
 284     specified in the specification file.
 285
 286     :param table: Table to generate.
 287     :param input_data: Data to process.
 288     :type table: pandas.Series
 289     :type input_data: InputData
 290     """
 291
 292     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 293
 294     # Transform the data
 295     logging.info(
 296         f"    Creating the data set for the {table.get(u'type', u'')} "
 297         f"{table.get(u'title', u'')}."
 298     )
 299     data = input_data.filter_data(table, continue_on_error=True)
 300     data = input_data.merge_data(data)
 301
 302     sort_tests = table.get(u"sort", None)
 303     if sort_tests:
 304         args = dict(
 305             inplace=True,
 306             ascending=(sort_tests == u"ascending")
 307         )
 308         data.sort_index(**args)
 309
 310     suites = input_data.filter_data(
 311         table, continue_on_error=True, data_set=u"suites")
 312     suites = input_data.merge_data(suites)
 313
 314     # Prepare the header of the tables
 315     header = list()
 316     for column in table[u"columns"]:
 317         header.append(
 318             u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
 319         )
 320
 321     for suite in suites.values:
 322         # Generate data
 323         suite_name = suite[u"name"]
 324         table_lst = list()
 325         for test in data.keys():
 326             if data[test][u"parent"] not in suite_name:
 327                 continue
 328             row_lst = list()
 329             for column in table[u"columns"]:
 330                 try:
 331                     col_data = str(data[test][column[
 332                         u"data"].split(u" ")[1]]).replace(u'"', u'""')
 333                     # Do not include tests with "Test Failed" in test message
 334                     if u"Test Failed" in col_data:
 335                         continue
 336                     col_data = col_data.replace(
 337                         u"No Data", u"Not Captured     "
 338                     )
 339                     if column[u"data"].split(u" ")[1] in (u"name", ):
 340                         if len(col_data) > 30:
 341                             col_data_lst = col_data.split(u"-")
 342                             half = int(len(col_data_lst) / 2)
 343                             col_data = f"{u'-'.join(col_data_lst[:half])}" \
 344                                        f"- |br| " \
 345                                        f"{u'-'.join(col_data_lst[half:])}"
 346                         col_data = f" |prein| {col_data} |preout| "
 347                     elif column[u"data"].split(u" ")[1] in (u"msg", ):
 348                         # Temporary solution: remove NDR results from message:
 349                         if bool(table.get(u'remove-ndr', False)):
 350                             try:
 351                                 col_data = col_data.split(u" |br| ", 1)[1]
 352                             except IndexError:
 353                                 pass
 354                         col_data = f" |prein| {col_data} |preout| "
 355                     elif column[u"data"].split(u" ")[1] in \
 356                             (u"conf-history", u"show-run"):
 357                         col_data = col_data.replace(u" |br| ", u"", 1)
 358                         col_data = f" |prein| {col_data[:-5]} |preout| "
 359                     row_lst.append(f'"{col_data}"')
 360                 except KeyError:
 361                     row_lst.append(u'"Not captured"')
 362             if len(row_lst) == len(table[u"columns"]):
 363                 table_lst.append(row_lst)
 364
 365         # Write the data to file
 366         if table_lst:
 367             separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
 368             file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
 369             logging.info(f"      Writing file: {file_name}")
 370             with open(file_name, u"wt") as file_handler:
 371                 file_handler.write(u",".join(header) + u"\n")
 372                 for item in table_lst:
 373                     file_handler.write(u",".join(item) + u"\n")
 374
 375     logging.info(u"  Done.")
 376
 377
 378 def _tpc_modify_test_name(test_name, ignore_nic=False):
 379     """Modify a test name by replacing its parts.
 380
 381     :param test_name: Test name to be modified.
 382     :param ignore_nic: If True, NIC is removed from TC name.
 383     :type test_name: str
 384     :type ignore_nic: bool
 385     :returns: Modified test name.
 386     :rtype: str
 387     """
 388     test_name_mod = test_name.\
 389         replace(u"-ndrpdrdisc", u""). \
 390         replace(u"-ndrpdr", u"").\
 391         replace(u"-pdrdisc", u""). \
 392         replace(u"-ndrdisc", u"").\
 393         replace(u"-pdr", u""). \
 394         replace(u"-ndr", u""). \
 395         replace(u"1t1c", u"1c").\
 396         replace(u"2t1c", u"1c"). \
 397         replace(u"2t2c", u"2c").\
 398         replace(u"4t2c", u"2c"). \
 399         replace(u"4t4c", u"4c").\
 400         replace(u"8t4c", u"4c")
 401
 402     if ignore_nic:
 403         return re.sub(REGEX_NIC, u"", test_name_mod)
 404     return test_name_mod
 405
 406
 407 def _tpc_modify_displayed_test_name(test_name):
 408     """Modify a test name which is displayed in a table by replacing its parts.
 409
 410     :param test_name: Test name to be modified.
 411     :type test_name: str
 412     :returns: Modified test name.
 413     :rtype: str
 414     """
 415     return test_name.\
 416         replace(u"1t1c", u"1c").\
 417         replace(u"2t1c", u"1c"). \
 418         replace(u"2t2c", u"2c").\
 419         replace(u"4t2c", u"2c"). \
 420         replace(u"4t4c", u"4c").\
 421         replace(u"8t4c", u"4c")
 422
 423
 424 def _tpc_insert_data(target, src, include_tests):
 425     """Insert src data to the target structure.
 426
 427     :param target: Target structure where the data is placed.
 428     :param src: Source data to be placed into the target stucture.
 429     :param include_tests: Which results will be included (MRR, NDR, PDR).
 430     :type target: list
 431     :type src: dict
 432     :type include_tests: str
 433     """
 434     try:
 435         if include_tests == u"MRR":
 436             target.append(
 437                 (
 438                     src[u"result"][u"receive-rate"],
 439                     src[u"result"][u"receive-stdev"]
 440                 )
 441             )
 442         elif include_tests == u"PDR":
 443             target.append(src[u"throughput"][u"PDR"][u"LOWER"])
 444         elif include_tests == u"NDR":
 445             target.append(src[u"throughput"][u"NDR"][u"LOWER"])
 446     except (KeyError, TypeError):
 447         pass
 448
 449
 450 def _tpc_sort_table(table):
 451     """Sort the table this way:
 452
 453     1. Put "New in CSIT-XXXX" at the first place.
 454     2. Put "See footnote" at the second place.
 455     3. Sort the rest by "Delta".
 456
 457     :param table: Table to sort.
 458     :type table: list
 459     :returns: Sorted table.
 460     :rtype: list
 461     """
 462
 463     tbl_new = list()
 464     tbl_see = list()
 465     tbl_delta = list()
 466     for item in table:
 467         if isinstance(item[-1], str):
 468             if u"New in CSIT" in item[-1]:
 469                 tbl_new.append(item)
 470             elif u"See footnote" in item[-1]:
 471                 tbl_see.append(item)
 472         else:
 473             tbl_delta.append(item)
 474
 475     # Sort the tables:
 476     tbl_new.sort(key=lambda rel: rel[0], reverse=False)
 477     tbl_see.sort(key=lambda rel: rel[0], reverse=False)
 478     tbl_see.sort(key=lambda rel: rel[-2], reverse=False)
 479     tbl_delta.sort(key=lambda rel: rel[0], reverse=False)
 480     tbl_delta.sort(key=lambda rel: rel[-2], reverse=True)
 481
 482     # Put the tables together:
 483     table = list()
 484     # We do not want "New in CSIT":
 485     # table.extend(tbl_new)
 486     table.extend(tbl_see)
 487     table.extend(tbl_delta)
 488
 489     return table
 490
 491
 492 def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
 493                              footnote=u"", sort_data=True, title=u""):
 494     """Generate html table from input data with simple sorting possibility.
 495
 496     :param header: Table header.
 497     :param data: Input data to be included in the table. It is a list of lists.
 498         Inner lists are rows in the table. All inner lists must be of the same
 499         length. The length of these lists must be the same as the length of the
 500         header.
 501     :param out_file_name: The name (relative or full path) where the
 502         generated html table is written.
 503     :param legend: The legend to display below the table.
 504     :param footnote: The footnote to display below the table (and legend).
 505     :param sort_data: If True the data sorting is enabled.
 506     :param title: The table (and file) title.
 507     :type header: list
 508     :type data: list of lists
 509     :type out_file_name: str
 510     :type legend: str
 511     :type footnote: str
 512     :type sort_data: bool
 513     :type title: str
 514     """
 515
 516     try:
 517         idx = header.index(u"Test Case")
 518     except ValueError:
 519         idx = 0
 520     params = {
 521         u"align-hdr": (
 522             [u"left", u"right"],
 523             [u"left", u"left", u"right"],
 524             [u"left", u"left", u"left", u"right"]
 525         ),
 526         u"align-itm": (
 527             [u"left", u"right"],
 528             [u"left", u"left", u"right"],
 529             [u"left", u"left", u"left", u"right"]
 530         ),
 531         u"width": ([28, 9], [4, 24, 10], [4, 4, 32, 10])
 532     }
 533
 534     df_data = pd.DataFrame(data, columns=header)
 535
 536     if sort_data:
 537         df_sorted = [df_data.sort_values(
 538             by=[key, header[idx]], ascending=[True, True]
 539             if key != header[idx] else [False, True]) for key in header]
 540         df_sorted_rev = [df_data.sort_values(
 541             by=[key, header[idx]], ascending=[False, True]
 542             if key != header[idx] else [True, True]) for key in header]
 543         df_sorted.extend(df_sorted_rev)
 544     else:
 545         df_sorted = df_data
 546
 547     fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
 548                    for idx in range(len(df_data))]]
 549     table_header = dict(
 550         values=[f"<b>{item.replace(u',', u',<br>')}</b>" for item in header],
 551         fill_color=u"#7eade7",
 552         align=params[u"align-hdr"][idx],
 553         font=dict(
 554             family=u"Courier New",
 555             size=13
 556         )
 557     )
 558
 559     fig = go.Figure()
 560
 561     if sort_data:
 562         for table in df_sorted:
 563             columns = [table.get(col) for col in header]
 564             fig.add_trace(
 565                 go.Table(
 566                     columnwidth=params[u"width"][idx],
 567                     header=table_header,
 568                     cells=dict(
 569                         values=columns,
 570                         fill_color=fill_color,
 571                         align=params[u"align-itm"][idx],
 572                         font=dict(
 573                             family=u"Courier New",
 574                             size=13
 575                         )
 576                     )
 577                 )
 578             )
 579
 580         buttons = list()
 581         menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
 582         menu_items_rev = [f"<b>{itm}</b> (descending)" for itm in header]
 583         menu_items.extend(menu_items_rev)
 584         for idx, hdr in enumerate(menu_items):
 585             visible = [False, ] * len(menu_items)
 586             visible[idx] = True
 587             buttons.append(
 588                 dict(
 589                     label=hdr.replace(u" [Mpps]", u""),
 590                     method=u"update",
 591                     args=[{u"visible": visible}],
 592                 )
 593             )
 594
 595         fig.update_layout(
 596             updatemenus=[
 597                 go.layout.Updatemenu(
 598                     type=u"dropdown",
 599                     direction=u"down",
 600                     x=0.0,
 601                     xanchor=u"left",
 602                     y=1.045,
 603                     yanchor=u"top",
 604                     active=len(menu_items) - 1,
 605                     buttons=list(buttons)
 606                 )
 607             ],
 608         )
 609     else:
 610         fig.add_trace(
 611             go.Table(
 612                 columnwidth=params[u"width"][idx],
 613                 header=table_header,
 614                 cells=dict(
 615                     values=[df_sorted.get(col) for col in header],
 616                     fill_color=fill_color,
 617                     align=params[u"align-itm"][idx],
 618                     font=dict(
 619                         family=u"Courier New",
 620                         size=13
 621                     )
 622                 )
 623             )
 624         )
 625
 626     ploff.plot(
 627         fig,
 628         show_link=False,
 629         auto_open=False,
 630         filename=f"{out_file_name}_in.html"
 631     )
 632
 633     file_name = out_file_name.split(u"/")[-1]
 634     if u"vpp" in out_file_name:
 635         path = u"_tmp/src/vpp_performance_tests/comparisons/"
 636     else:
 637         path = u"_tmp/src/dpdk_performance_tests/comparisons/"
 638     with open(f"{path}{file_name}.rst", u"wt") as rst_file:
 639         rst_file.write(
 640             u"\n"
 641             u".. |br| raw:: html\n\n    <br />\n\n\n"
 642             u".. |prein| raw:: html\n\n    <pre>\n\n\n"
 643             u".. |preout| raw:: html\n\n    </pre>\n\n"
 644         )
 645         if title:
 646             rst_file.write(f"{title}\n")
 647             rst_file.write(f"{u'~' * len(title)}\n\n")
 648         rst_file.write(
 649             u".. raw:: html\n\n"
 650             f'    <iframe frameborder="0" scrolling="no" '
 651             f'width="1600" height="1200" '
 652             f'src="../..{out_file_name.replace(u"_build", u"")}_in.html">'
 653             f'</iframe>\n\n'
 654         )
 655         if legend:
 656             rst_file.write(legend[1:].replace(u"\n", u" |br| "))
 657         if footnote:
 658             rst_file.write(footnote.replace(u"\n", u" |br| ")[1:])
 659
 660
 661 def table_perf_comparison(table, input_data):
 662     """Generate the table(s) with algorithm: table_perf_comparison
 663     specified in the specification file.
 664
 665     :param table: Table to generate.
 666     :param input_data: Data to process.
 667     :type table: pandas.Series
 668     :type input_data: InputData
 669     """
 670
 671     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 672
 673     # Transform the data
 674     logging.info(
 675         f"    Creating the data set for the {table.get(u'type', u'')} "
 676         f"{table.get(u'title', u'')}."
 677     )
 678     data = input_data.filter_data(table, continue_on_error=True)
 679
 680     # Prepare the header of the tables
 681     try:
 682         header = [u"Test Case", ]
 683         legend = u"\nLegend:\n"
 684
 685         rca_data = None
 686         rca = table.get(u"rca", None)
 687         if rca:
 688             try:
 689                 with open(rca.get(u"data-file", u""), u"r") as rca_file:
 690                     rca_data = load(rca_file, Loader=FullLoader)
 691                 header.insert(0, rca.get(u"title", u"RCA"))
 692                 legend += (
 693                     u"RCA: Reference to the Root Cause Analysis, see below.\n"
 694                 )
 695             except (YAMLError, IOError) as err:
 696                 logging.warning(repr(err))
 697
 698         history = table.get(u"history", list())
 699         for item in history:
 700             header.extend(
 701                 [
 702                     f"{item[u'title']} Avg({table[u'include-tests']})",
 703                     f"{item[u'title']} Stdev({table[u'include-tests']})"
 704                 ]
 705             )
 706             legend += (
 707                 f"{item[u'title']} Avg({table[u'include-tests']}): "
 708                 f"Mean value of {table[u'include-tests']} [Mpps] computed from "
 709                 f"a series of runs of the listed tests executed against "
 710                 f"{item[u'title']}.\n"
 711                 f"{item[u'title']} Stdev({table[u'include-tests']}): "
 712                 f"Standard deviation value of {table[u'include-tests']} [Mpps] "
 713                 f"computed from a series of runs of the listed tests executed "
 714                 f"against {item[u'title']}.\n"
 715             )
 716         header.extend(
 717             [
 718                 f"{table[u'reference'][u'title']} "
 719                 f"Avg({table[u'include-tests']})",
 720                 f"{table[u'reference'][u'title']} "
 721                 f"Stdev({table[u'include-tests']})",
 722                 f"{table[u'compare'][u'title']} "
 723                 f"Avg({table[u'include-tests']})",
 724                 f"{table[u'compare'][u'title']} "
 725                 f"Stdev({table[u'include-tests']})",
 726                 f"Diff({table[u'reference'][u'title']},"
 727                 f"{table[u'compare'][u'title']})",
 728                 u"Stdev(Diff)"
 729             ]
 730         )
 731         header_str = u";".join(header) + u"\n"
 732         legend += (
 733             f"{table[u'reference'][u'title']} "
 734             f"Avg({table[u'include-tests']}): "
 735             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
 736             f"series of runs of the listed tests executed against "
 737             f"{table[u'reference'][u'title']}.\n"
 738             f"{table[u'reference'][u'title']} "
 739             f"Stdev({table[u'include-tests']}): "
 740             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
 741             f"computed from a series of runs of the listed tests executed "
 742             f"against {table[u'reference'][u'title']}.\n"
 743             f"{table[u'compare'][u'title']} "
 744             f"Avg({table[u'include-tests']}): "
 745             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
 746             f"series of runs of the listed tests executed against "
 747             f"{table[u'compare'][u'title']}.\n"
 748             f"{table[u'compare'][u'title']} "
 749             f"Stdev({table[u'include-tests']}): "
 750             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
 751             f"computed from a series of runs of the listed tests executed "
 752             f"against {table[u'compare'][u'title']}.\n"
 753             f"Diff({table[u'reference'][u'title']},"
 754             f"{table[u'compare'][u'title']}): "
 755             f"Percentage change calculated for mean values.\n"
 756             u"Stdev(Diff): "
 757             u"Standard deviation of percentage change calculated for mean "
 758             u"values.\n"
 759             u"NT: Not Tested\n"
 760         )
 761     except (AttributeError, KeyError) as err:
 762         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 763         return
 764
 765     # Prepare data to the table:
 766     tbl_dict = dict()
 767     for job, builds in table[u"reference"][u"data"].items():
 768         for build in builds:
 769             for tst_name, tst_data in data[job][str(build)].items():
 770                 tst_name_mod = _tpc_modify_test_name(tst_name)
 771                 if (u"across topologies" in table[u"title"].lower() or
 772                         (u" 3n-" in table[u"title"].lower() and
 773                          u" 2n-" in table[u"title"].lower())):
 774                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 775                 if tbl_dict.get(tst_name_mod, None) is None:
 776                     name = tst_data[u'name'].rsplit(u'-', 1)[0]
 777                     if u"across testbeds" in table[u"title"].lower() or \
 778                             u"across topologies" in table[u"title"].lower():
 779                         name = _tpc_modify_displayed_test_name(name)
 780                     tbl_dict[tst_name_mod] = {
 781                         u"name": name,
 782                         u"replace-ref": True,
 783                         u"replace-cmp": True,
 784                         u"ref-data": list(),
 785                         u"cmp-data": list()
 786                     }
 787                 _tpc_insert_data(target=tbl_dict[tst_name_mod][u"ref-data"],
 788                                  src=tst_data,
 789                                  include_tests=table[u"include-tests"])
 790
 791     replacement = table[u"reference"].get(u"data-replacement", None)
 792     if replacement:
 793         rpl_data = input_data.filter_data(
 794             table, data=replacement, continue_on_error=True)
 795         for job, builds in replacement.items():
 796             for build in builds:
 797                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 798                     tst_name_mod = _tpc_modify_test_name(tst_name)
 799                     if (u"across topologies" in table[u"title"].lower() or
 800                             (u" 3n-" in table[u"title"].lower() and
 801                              u" 2n-" in table[u"title"].lower())):
 802                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 803                     if tbl_dict.get(tst_name_mod, None) is None:
 804                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
 805                         if u"across testbeds" in table[u"title"].lower() or \
 806                                 u"across topologies" in table[u"title"].lower():
 807                             name = _tpc_modify_displayed_test_name(name)
 808                         tbl_dict[tst_name_mod] = {
 809                             u"name": name,
 810                             u"replace-ref": False,
 811                             u"replace-cmp": True,
 812                             u"ref-data": list(),
 813                             u"cmp-data": list()
 814                         }
 815                     if tbl_dict[tst_name_mod][u"replace-ref"]:
 816                         tbl_dict[tst_name_mod][u"replace-ref"] = False
 817                         tbl_dict[tst_name_mod][u"ref-data"] = list()
 818
 819                     _tpc_insert_data(
 820                         target=tbl_dict[tst_name_mod][u"ref-data"],
 821                         src=tst_data,
 822                         include_tests=table[u"include-tests"]
 823                     )
 824
 825     for job, builds in table[u"compare"][u"data"].items():
 826         for build in builds:
 827             for tst_name, tst_data in data[job][str(build)].items():
 828                 tst_name_mod = _tpc_modify_test_name(tst_name)
 829                 if (u"across topologies" in table[u"title"].lower() or
 830                         (u" 3n-" in table[u"title"].lower() and
 831                          u" 2n-" in table[u"title"].lower())):
 832                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 833                 if tbl_dict.get(tst_name_mod, None) is None:
 834                     name = tst_data[u'name'].rsplit(u'-', 1)[0]
 835                     if u"across testbeds" in table[u"title"].lower() or \
 836                             u"across topologies" in table[u"title"].lower():
 837                         name = _tpc_modify_displayed_test_name(name)
 838                     tbl_dict[tst_name_mod] = {
 839                         u"name": name,
 840                         u"replace-ref": False,
 841                         u"replace-cmp": True,
 842                         u"ref-data": list(),
 843                         u"cmp-data": list()
 844                     }
 845                 _tpc_insert_data(
 846                     target=tbl_dict[tst_name_mod][u"cmp-data"],
 847                     src=tst_data,
 848                     include_tests=table[u"include-tests"]
 849                 )
 850
 851     replacement = table[u"compare"].get(u"data-replacement", None)
 852     if replacement:
 853         rpl_data = input_data.filter_data(
 854             table, data=replacement, continue_on_error=True)
 855         for job, builds in replacement.items():
 856             for build in builds:
 857                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 858                     tst_name_mod = _tpc_modify_test_name(tst_name)
 859                     if (u"across topologies" in table[u"title"].lower() or
 860                             (u" 3n-" in table[u"title"].lower() and
 861                              u" 2n-" in table[u"title"].lower())):
 862                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 863                     if tbl_dict.get(tst_name_mod, None) is None:
 864                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
 865                         if u"across testbeds" in table[u"title"].lower() or \
 866                                 u"across topologies" in table[u"title"].lower():
 867                             name = _tpc_modify_displayed_test_name(name)
 868                         tbl_dict[tst_name_mod] = {
 869                             u"name": name,
 870                             u"replace-ref": False,
 871                             u"replace-cmp": False,
 872                             u"ref-data": list(),
 873                             u"cmp-data": list()
 874                         }
 875                     if tbl_dict[tst_name_mod][u"replace-cmp"]:
 876                         tbl_dict[tst_name_mod][u"replace-cmp"] = False
 877                         tbl_dict[tst_name_mod][u"cmp-data"] = list()
 878
 879                     _tpc_insert_data(
 880                         target=tbl_dict[tst_name_mod][u"cmp-data"],
 881                         src=tst_data,
 882                         include_tests=table[u"include-tests"]
 883                     )
 884
 885     for item in history:
 886         for job, builds in item[u"data"].items():
 887             for build in builds:
 888                 for tst_name, tst_data in data[job][str(build)].items():
 889                     tst_name_mod = _tpc_modify_test_name(tst_name)
 890                     if (u"across topologies" in table[u"title"].lower() or
 891                             (u" 3n-" in table[u"title"].lower() and
 892                              u" 2n-" in table[u"title"].lower())):
 893                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 894                     if tbl_dict.get(tst_name_mod, None) is None:
 895                         continue
 896                     if tbl_dict[tst_name_mod].get(u"history", None) is None:
 897                         tbl_dict[tst_name_mod][u"history"] = OrderedDict()
 898                     if tbl_dict[tst_name_mod][u"history"].\
 899                             get(item[u"title"], None) is None:
 900                         tbl_dict[tst_name_mod][u"history"][item[
 901                             u"title"]] = list()
 902                     try:
 903                         if table[u"include-tests"] == u"MRR":
 904                             res = (tst_data[u"result"][u"receive-rate"],
 905                                    tst_data[u"result"][u"receive-stdev"])
 906                         elif table[u"include-tests"] == u"PDR":
 907                             res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
 908                         elif table[u"include-tests"] == u"NDR":
 909                             res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
 910                         else:
 911                             continue
 912                         tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
 913                             append(res)
 914                     except (TypeError, KeyError):
 915                         pass
 916
 917     tbl_lst = list()
 918     for tst_name in tbl_dict:
 919         item = [tbl_dict[tst_name][u"name"], ]
 920         if history:
 921             if tbl_dict[tst_name].get(u"history", None) is not None:
 922                 for hist_data in tbl_dict[tst_name][u"history"].values():
 923                     if hist_data:
 924                         if table[u"include-tests"] == u"MRR":
 925                             item.append(round(hist_data[0][0] / 1e6, 1))
 926                             item.append(round(hist_data[0][1] / 1e6, 1))
 927                         else:
 928                             item.append(round(mean(hist_data) / 1e6, 1))
 929                             item.append(round(stdev(hist_data) / 1e6, 1))
 930                     else:
 931                         item.extend([u"NT", u"NT"])
 932             else:
 933                 item.extend([u"NT", u"NT"])
 934         data_r = tbl_dict[tst_name][u"ref-data"]
 935         if data_r:
 936             if table[u"include-tests"] == u"MRR":
 937                 data_r_mean = data_r[0][0]
 938                 data_r_stdev = data_r[0][1]
 939             else:
 940                 data_r_mean = mean(data_r)
 941                 data_r_stdev = stdev(data_r)
 942             item.append(round(data_r_mean / 1e6, 1))
 943             item.append(round(data_r_stdev / 1e6, 1))
 944         else:
 945             data_r_mean = None
 946             data_r_stdev = None
 947             item.extend([u"NT", u"NT"])
 948         data_c = tbl_dict[tst_name][u"cmp-data"]
 949         if data_c:
 950             if table[u"include-tests"] == u"MRR":
 951                 data_c_mean = data_c[0][0]
 952                 data_c_stdev = data_c[0][1]
 953             else:
 954                 data_c_mean = mean(data_c)
 955                 data_c_stdev = stdev(data_c)
 956             item.append(round(data_c_mean / 1e6, 1))
 957             item.append(round(data_c_stdev / 1e6, 1))
 958         else:
 959             data_c_mean = None
 960             data_c_stdev = None
 961             item.extend([u"NT", u"NT"])
 962         if item[-2] == u"NT":
 963             pass
 964         elif item[-4] == u"NT":
 965             item.append(u"New in CSIT-2001")
 966             item.append(u"New in CSIT-2001")
 967         elif data_r_mean is not None and data_c_mean is not None:
 968             delta, d_stdev = relative_change_stdev(
 969                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
 970             )
 971             try:
 972                 item.append(round(delta))
 973             except ValueError:
 974                 item.append(delta)
 975             try:
 976                 item.append(round(d_stdev))
 977             except ValueError:
 978                 item.append(d_stdev)
 979         if rca_data:
 980             rca_nr = rca_data.get(item[0], u"-")
 981             item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-")
 982         if (len(item) == len(header)) and (item[-4] != u"NT"):
 983             tbl_lst.append(item)
 984
 985     tbl_lst = _tpc_sort_table(tbl_lst)
 986
 987     # Generate csv tables:
 988     csv_file = f"{table[u'output-file']}.csv"
 989     with open(csv_file, u"wt") as file_handler:
 990         file_handler.write(header_str)
 991         for test in tbl_lst:
 992             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 993
 994     txt_file_name = f"{table[u'output-file']}.txt"
 995     convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
 996
 997     footnote = u""
 998     with open(txt_file_name, u'a') as txt_file:
 999         txt_file.write(legend)
1000         if rca_data:
1001             footnote = rca_data.get(u"footnote", u"")
1002             if footnote:
1003                 txt_file.write(footnote)
1004         txt_file.write(u":END")
1005
1006     # Generate html table:
1007     _tpc_generate_html_table(
1008         header,
1009         tbl_lst,
1010         table[u'output-file'],
1011         legend=legend,
1012         footnote=footnote
1013     )
1014
1015
1016 def table_perf_comparison_nic(table, input_data):
1017     """Generate the table(s) with algorithm: table_perf_comparison
1018     specified in the specification file.
1019
1020     :param table: Table to generate.
1021     :param input_data: Data to process.
1022     :type table: pandas.Series
1023     :type input_data: InputData
1024     """
1025
1026     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1027
1028     # Transform the data
1029     logging.info(
1030         f"    Creating the data set for the {table.get(u'type', u'')} "
1031         f"{table.get(u'title', u'')}."
1032     )
1033     data = input_data.filter_data(table, continue_on_error=True)
1034
1035     # Prepare the header of the tables
1036     try:
1037         header = [u"Test Case", ]
1038         legend = u"\nLegend:\n"
1039
1040         rca_data = None
1041         rca = table.get(u"rca", None)
1042         if rca:
1043             try:
1044                 with open(rca.get(u"data-file", ""), u"r") as rca_file:
1045                     rca_data = load(rca_file, Loader=FullLoader)
1046                 header.insert(0, rca.get(u"title", "RCA"))
1047                 legend += (
1048                     u"RCA: Reference to the Root Cause Analysis, see below.\n"
1049                 )
1050             except (YAMLError, IOError) as err:
1051                 logging.warning(repr(err))
1052
1053         history = table.get(u"history", list())
1054         for item in history:
1055             header.extend(
1056                 [
1057                     f"{item[u'title']} Avg({table[u'include-tests']})",
1058                     f"{item[u'title']} Stdev({table[u'include-tests']})"
1059                 ]
1060             )
1061             legend += (
1062                 f"{item[u'title']} Avg({table[u'include-tests']}): "
1063                 f"Mean value of {table[u'include-tests']} [Mpps] computed from "
1064                 f"a series of runs of the listed tests executed against "
1065                 f"{item[u'title']}.\n"
1066                 f"{item[u'title']} Stdev({table[u'include-tests']}): "
1067                 f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1068                 f"computed from a series of runs of the listed tests executed "
1069                 f"against {item[u'title']}.\n"
1070             )
1071         header.extend(
1072             [
1073                 f"{table[u'reference'][u'title']} "
1074                 f"Avg({table[u'include-tests']})",
1075                 f"{table[u'reference'][u'title']} "
1076                 f"Stdev({table[u'include-tests']})",
1077                 f"{table[u'compare'][u'title']} "
1078                 f"Avg({table[u'include-tests']})",
1079                 f"{table[u'compare'][u'title']} "
1080                 f"Stdev({table[u'include-tests']})",
1081                 f"Diff({table[u'reference'][u'title']},"
1082                 f"{table[u'compare'][u'title']})",
1083                 u"Stdev(Diff)"
1084             ]
1085         )
1086         header_str = u";".join(header) + u"\n"
1087         legend += (
1088             f"{table[u'reference'][u'title']} "
1089             f"Avg({table[u'include-tests']}): "
1090             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
1091             f"series of runs of the listed tests executed against "
1092             f"{table[u'reference'][u'title']}.\n"
1093             f"{table[u'reference'][u'title']} "
1094             f"Stdev({table[u'include-tests']}): "
1095             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1096             f"computed from a series of runs of the listed tests executed "
1097             f"against {table[u'reference'][u'title']}.\n"
1098             f"{table[u'compare'][u'title']} "
1099             f"Avg({table[u'include-tests']}): "
1100             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
1101             f"series of runs of the listed tests executed against "
1102             f"{table[u'compare'][u'title']}.\n"
1103             f"{table[u'compare'][u'title']} "
1104             f"Stdev({table[u'include-tests']}): "
1105             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1106             f"computed from a series of runs of the listed tests executed "
1107             f"against {table[u'compare'][u'title']}.\n"
1108             f"Diff({table[u'reference'][u'title']},"
1109             f"{table[u'compare'][u'title']}): "
1110             f"Percentage change calculated for mean values.\n"
1111             u"Stdev(Diff): "
1112             u"Standard deviation of percentage change calculated for mean "
1113             u"values.\n"
1114             u"NT: Not Tested\n"
1115         )
1116     except (AttributeError, KeyError) as err:
1117         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1118         return
1119
1120     # Prepare data to the table:
1121     tbl_dict = dict()
1122     for job, builds in table[u"reference"][u"data"].items():
1123         for build in builds:
1124             for tst_name, tst_data in data[job][str(build)].items():
1125                 if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
1126                     continue
1127                 tst_name_mod = _tpc_modify_test_name(tst_name, ignore_nic=True)
1128                 if (u"across topologies" in table[u"title"].lower() or
1129                         (u" 3n-" in table[u"title"].lower() and
1130                          u" 2n-" in table[u"title"].lower())):
1131                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1132                 if tbl_dict.get(tst_name_mod, None) is None:
1133                     name = tst_data[u'name'].rsplit(u'-', 1)[0]
1134                     if u"across testbeds" in table[u"title"].lower() or \
1135                             u"across topologies" in table[u"title"].lower():
1136                         name = _tpc_modify_displayed_test_name(name)
1137                     tbl_dict[tst_name_mod] = {
1138                         u"name": name,
1139                         u"replace-ref": True,
1140                         u"replace-cmp": True,
1141                         u"ref-data": list(),
1142                         u"cmp-data": list()
1143                     }
1144                 _tpc_insert_data(
1145                     target=tbl_dict[tst_name_mod][u"ref-data"],
1146                     src=tst_data,
1147                     include_tests=table[u"include-tests"]
1148                 )
1149
1150     replacement = table[u"reference"].get(u"data-replacement", None)
1151     if replacement:
1152         rpl_data = input_data.filter_data(
1153             table, data=replacement, continue_on_error=True)
1154         for job, builds in replacement.items():
1155             for build in builds:
1156                 for tst_name, tst_data in rpl_data[job][str(build)].items():
1157                     if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
1158                         continue
1159                     tst_name_mod = \
1160                         _tpc_modify_test_name(tst_name, ignore_nic=True)
1161                     if (u"across topologies" in table[u"title"].lower() or
1162                             (u" 3n-" in table[u"title"].lower() and
1163                              u" 2n-" in table[u"title"].lower())):
1164                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1165                     if tbl_dict.get(tst_name_mod, None) is None:
1166                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
1167                         if u"across testbeds" in table[u"title"].lower() or \
1168                                 u"across topologies" in table[u"title"].lower():
1169                             name = _tpc_modify_displayed_test_name(name)
1170                         tbl_dict[tst_name_mod] = {
1171                             u"name": name,
1172                             u"replace-ref": False,
1173                             u"replace-cmp": True,
1174                             u"ref-data": list(),
1175                             u"cmp-data": list()
1176                         }
1177                     if tbl_dict[tst_name_mod][u"replace-ref"]:
1178                         tbl_dict[tst_name_mod][u"replace-ref"] = False
1179                         tbl_dict[tst_name_mod][u"ref-data"] = list()
1180
1181                     _tpc_insert_data(
1182                         target=tbl_dict[tst_name_mod][u"ref-data"],
1183                         src=tst_data,
1184                         include_tests=table[u"include-tests"]
1185                     )
1186
1187     for job, builds in table[u"compare"][u"data"].items():
1188         for build in builds:
1189             for tst_name, tst_data in data[job][str(build)].items():
1190                 if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
1191                     continue
1192                 tst_name_mod = _tpc_modify_test_name(tst_name, ignore_nic=True)
1193                 if (u"across topologies" in table[u"title"].lower() or
1194                         (u" 3n-" in table[u"title"].lower() and
1195                          u" 2n-" in table[u"title"].lower())):
1196                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1197                 if tbl_dict.get(tst_name_mod, None) is None:
1198                     name = tst_data[u'name'].rsplit(u'-', 1)[0]
1199                     if u"across testbeds" in table[u"title"].lower() or \
1200                             u"across topologies" in table[u"title"].lower():
1201                         name = _tpc_modify_displayed_test_name(name)
1202                     tbl_dict[tst_name_mod] = {
1203                         u"name": name,
1204                         u"replace-ref": False,
1205                         u"replace-cmp": True,
1206                         u"ref-data": list(),
1207                         u"cmp-data": list()
1208                     }
1209                 _tpc_insert_data(
1210                     target=tbl_dict[tst_name_mod][u"cmp-data"],
1211                     src=tst_data,
1212                     include_tests=table[u"include-tests"]
1213                 )
1214
1215     replacement = table[u"compare"].get(u"data-replacement", None)
1216     if replacement:
1217         rpl_data = input_data.filter_data(
1218             table, data=replacement, continue_on_error=True)
1219         for job, builds in replacement.items():
1220             for build in builds:
1221                 for tst_name, tst_data in rpl_data[job][str(build)].items():
1222                     if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
1223                         continue
1224                     tst_name_mod = \
1225                         _tpc_modify_test_name(tst_name, ignore_nic=True)
1226                     if (u"across topologies" in table[u"title"].lower() or
1227                             (u" 3n-" in table[u"title"].lower() and
1228                              u" 2n-" in table[u"title"].lower())):
1229                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1230                     if tbl_dict.get(tst_name_mod, None) is None:
1231                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
1232                         if u"across testbeds" in table[u"title"].lower() or \
1233                                 u"across topologies" in table[u"title"].lower():
1234                             name = _tpc_modify_displayed_test_name(name)
1235                         tbl_dict[tst_name_mod] = {
1236                             u"name": name,
1237                             u"replace-ref": False,
1238                             u"replace-cmp": False,
1239                             u"ref-data": list(),
1240                             u"cmp-data": list()
1241                         }
1242                     if tbl_dict[tst_name_mod][u"replace-cmp"]:
1243                         tbl_dict[tst_name_mod][u"replace-cmp"] = False
1244                         tbl_dict[tst_name_mod][u"cmp-data"] = list()
1245
1246                     _tpc_insert_data(
1247                         target=tbl_dict[tst_name_mod][u"cmp-data"],
1248                         src=tst_data,
1249                         include_tests=table[u"include-tests"]
1250                     )
1251
1252     for item in history:
1253         for job, builds in item[u"data"].items():
1254             for build in builds:
1255                 for tst_name, tst_data in data[job][str(build)].items():
1256                     if item[u"nic"] not in tst_data[u"tags"]:
1257                         continue
1258                     tst_name_mod = \
1259                         _tpc_modify_test_name(tst_name, ignore_nic=True)
1260                     if (u"across topologies" in table[u"title"].lower() or
1261                             (u" 3n-" in table[u"title"].lower() and
1262                              u" 2n-" in table[u"title"].lower())):
1263                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1264                     if tbl_dict.get(tst_name_mod, None) is None:
1265                         continue
1266                     if tbl_dict[tst_name_mod].get(u"history", None) is None:
1267                         tbl_dict[tst_name_mod][u"history"] = OrderedDict()
1268                     if tbl_dict[tst_name_mod][u"history"].\
1269                             get(item[u"title"], None) is None:
1270                         tbl_dict[tst_name_mod][u"history"][item[
1271                             u"title"]] = list()
1272                     try:
1273                         if table[u"include-tests"] == u"MRR":
1274                             res = (tst_data[u"result"][u"receive-rate"],
1275                                    tst_data[u"result"][u"receive-stdev"])
1276                         elif table[u"include-tests"] == u"PDR":
1277                             res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
1278                         elif table[u"include-tests"] == u"NDR":
1279                             res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
1280                         else:
1281                             continue
1282                         tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
1283                             append(res)
1284                     except (TypeError, KeyError):
1285                         pass
1286
1287     tbl_lst = list()
1288     for tst_name in tbl_dict:
1289         item = [tbl_dict[tst_name][u"name"], ]
1290         if history:
1291             if tbl_dict[tst_name].get(u"history", None) is not None:
1292                 for hist_data in tbl_dict[tst_name][u"history"].values():
1293                     if hist_data:
1294                         if table[u"include-tests"] == u"MRR":
1295                             item.append(round(hist_data[0][0] / 1e6, 1))
1296                             item.append(round(hist_data[0][1] / 1e6, 1))
1297                         else:
1298                             item.append(round(mean(hist_data) / 1e6, 1))
1299                             item.append(round(stdev(hist_data) / 1e6, 1))
1300                     else:
1301                         item.extend([u"NT", u"NT"])
1302             else:
1303                 item.extend([u"NT", u"NT"])
1304         data_r = tbl_dict[tst_name][u"ref-data"]
1305         if data_r:
1306             if table[u"include-tests"] == u"MRR":
1307                 data_r_mean = data_r[0][0]
1308                 data_r_stdev = data_r[0][1]
1309             else:
1310                 data_r_mean = mean(data_r)
1311                 data_r_stdev = stdev(data_r)
1312             item.append(round(data_r_mean / 1e6, 1))
1313             item.append(round(data_r_stdev / 1e6, 1))
1314         else:
1315             data_r_mean = None
1316             data_r_stdev = None
1317             item.extend([u"NT", u"NT"])
1318         data_c = tbl_dict[tst_name][u"cmp-data"]
1319         if data_c:
1320             if table[u"include-tests"] == u"MRR":
1321                 data_c_mean = data_c[0][0]
1322                 data_c_stdev = data_c[0][1]
1323             else:
1324                 data_c_mean = mean(data_c)
1325                 data_c_stdev = stdev(data_c)
1326             item.append(round(data_c_mean / 1e6, 1))
1327             item.append(round(data_c_stdev / 1e6, 1))
1328         else:
1329             data_c_mean = None
1330             data_c_stdev = None
1331             item.extend([u"NT", u"NT"])
1332         if item[-2] == u"NT":
1333             pass
1334         elif item[-4] == u"NT":
1335             item.append(u"New in CSIT-2001")
1336             item.append(u"New in CSIT-2001")
1337         elif data_r_mean is not None and data_c_mean is not None:
1338             delta, d_stdev = relative_change_stdev(
1339                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
1340             )
1341             try:
1342                 item.append(round(delta))
1343             except ValueError:
1344                 item.append(delta)
1345             try:
1346                 item.append(round(d_stdev))
1347             except ValueError:
1348                 item.append(d_stdev)
1349         if rca_data:
1350             rca_nr = rca_data.get(item[0], u"-")
1351             item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1352         if (len(item) == len(header)) and (item[-4] != u"NT"):
1353             tbl_lst.append(item)
1354
1355     tbl_lst = _tpc_sort_table(tbl_lst)
1356
1357     # Generate csv tables:
1358     csv_file = f"{table[u'output-file']}.csv"
1359     with open(csv_file, u"wt") as file_handler:
1360         file_handler.write(header_str)
1361         for test in tbl_lst:
1362             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1363
1364     txt_file_name = f"{table[u'output-file']}.txt"
1365     convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
1366
1367     footnote = u""
1368     with open(txt_file_name, u'a') as txt_file:
1369         txt_file.write(legend)
1370         if rca_data:
1371             footnote = rca_data.get(u"footnote", u"")
1372             if footnote:
1373                 txt_file.write(footnote)
1374         txt_file.write(u":END")
1375
1376     # Generate html table:
1377     _tpc_generate_html_table(
1378         header,
1379         tbl_lst,
1380         table[u'output-file'],
1381         legend=legend,
1382         footnote=footnote
1383     )
1384
1385
1386 def table_nics_comparison(table, input_data):
1387     """Generate the table(s) with algorithm: table_nics_comparison
1388     specified in the specification file.
1389
1390     :param table: Table to generate.
1391     :param input_data: Data to process.
1392     :type table: pandas.Series
1393     :type input_data: InputData
1394     """
1395
1396     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1397
1398     # Transform the data
1399     logging.info(
1400         f"    Creating the data set for the {table.get(u'type', u'')} "
1401         f"{table.get(u'title', u'')}."
1402     )
1403     data = input_data.filter_data(table, continue_on_error=True)
1404
1405     # Prepare the header of the tables
1406     try:
1407         header = [
1408             u"Test Case",
1409             f"{table[u'reference'][u'title']} "
1410             f"Avg({table[u'include-tests']})",
1411             f"{table[u'reference'][u'title']} "
1412             f"Stdev({table[u'include-tests']})",
1413             f"{table[u'compare'][u'title']} "
1414             f"Avg({table[u'include-tests']})",
1415             f"{table[u'compare'][u'title']} "
1416             f"Stdev({table[u'include-tests']})",
1417             f"Diff({table[u'reference'][u'title']},"
1418             f"{table[u'compare'][u'title']})",
1419             u"Stdev(Diff)"
1420         ]
1421         legend = (
1422             u"\nLegend:\n"
1423             f"{table[u'reference'][u'title']} "
1424             f"Avg({table[u'include-tests']}): "
1425             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
1426             f"series of runs of the listed tests executed using "
1427             f"{table[u'reference'][u'title']} NIC.\n"
1428             f"{table[u'reference'][u'title']} "
1429             f"Stdev({table[u'include-tests']}): "
1430             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1431             f"computed from a series of runs of the listed tests executed "
1432             f"using {table[u'reference'][u'title']} NIC.\n"
1433             f"{table[u'compare'][u'title']} "
1434             f"Avg({table[u'include-tests']}): "
1435             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
1436             f"series of runs of the listed tests executed using "
1437             f"{table[u'compare'][u'title']} NIC.\n"
1438             f"{table[u'compare'][u'title']} "
1439             f"Stdev({table[u'include-tests']}): "
1440             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1441             f"computed from a series of runs of the listed tests executed "
1442             f"using {table[u'compare'][u'title']} NIC.\n"
1443             f"Diff({table[u'reference'][u'title']},"
1444             f"{table[u'compare'][u'title']}): "
1445             f"Percentage change calculated for mean values.\n"
1446             u"Stdev(Diff): "
1447             u"Standard deviation of percentage change calculated for mean "
1448             u"values.\n"
1449             u":END"
1450         )
1451
1452     except (AttributeError, KeyError) as err:
1453         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1454         return
1455
1456     # Prepare data to the table:
1457     tbl_dict = dict()
1458     for job, builds in table[u"data"].items():
1459         for build in builds:
1460             for tst_name, tst_data in data[job][str(build)].items():
1461                 tst_name_mod = _tpc_modify_test_name(tst_name, ignore_nic=True)
1462                 if tbl_dict.get(tst_name_mod, None) is None:
1463                     name = tst_data[u'name'].rsplit(u'-', 1)[0]
1464                     tbl_dict[tst_name_mod] = {
1465                         u"name": name,
1466                         u"ref-data": list(),
1467                         u"cmp-data": list()
1468                     }
1469                 try:
1470                     if table[u"include-tests"] == u"MRR":
1471                         result = (tst_data[u"result"][u"receive-rate"],
1472                                   tst_data[u"result"][u"receive-stdev"])
1473                     elif table[u"include-tests"] == u"PDR":
1474                         result = tst_data[u"throughput"][u"PDR"][u"LOWER"]
1475                     elif table[u"include-tests"] == u"NDR":
1476                         result = tst_data[u"throughput"][u"NDR"][u"LOWER"]
1477                     else:
1478                         continue
1479
1480                     if result and \
1481                             table[u"reference"][u"nic"] in tst_data[u"tags"]:
1482                         tbl_dict[tst_name_mod][u"ref-data"].append(result)
1483                     elif result and \
1484                             table[u"compare"][u"nic"] in tst_data[u"tags"]:
1485                         tbl_dict[tst_name_mod][u"cmp-data"].append(result)
1486                 except (TypeError, KeyError) as err:
1487                     logging.debug(f"No data for {tst_name}\n{repr(err)}")
1488                     # No data in output.xml for this test
1489
1490     tbl_lst = list()
1491     for tst_name in tbl_dict:
1492         item = [tbl_dict[tst_name][u"name"], ]
1493         data_r = tbl_dict[tst_name][u"ref-data"]
1494         if data_r:
1495             if table[u"include-tests"] == u"MRR":
1496                 data_r_mean = data_r[0][0]
1497                 data_r_stdev = data_r[0][1]
1498             else:
1499                 data_r_mean = mean(data_r)
1500                 data_r_stdev = stdev(data_r)
1501             item.append(round(data_r_mean / 1e6, 1))
1502             item.append(round(data_r_stdev / 1e6, 1))
1503         else:
1504             data_r_mean = None
1505             data_r_stdev = None
1506             item.extend([None, None])
1507         data_c = tbl_dict[tst_name][u"cmp-data"]
1508         if data_c:
1509             if table[u"include-tests"] == u"MRR":
1510                 data_c_mean = data_c[0][0]
1511                 data_c_stdev = data_c[0][1]
1512             else:
1513                 data_c_mean = mean(data_c)
1514                 data_c_stdev = stdev(data_c)
1515             item.append(round(data_c_mean / 1e6, 1))
1516             item.append(round(data_c_stdev / 1e6, 1))
1517         else:
1518             data_c_mean = None
1519             data_c_stdev = None
1520             item.extend([None, None])
1521         if data_r_mean is not None and data_c_mean is not None:
1522             delta, d_stdev = relative_change_stdev(
1523                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
1524             )
1525             try:
1526                 item.append(round(delta))
1527             except ValueError:
1528                 item.append(delta)
1529             try:
1530                 item.append(round(d_stdev))
1531             except ValueError:
1532                 item.append(d_stdev)
1533             tbl_lst.append(item)
1534
1535     # Sort the table according to the relative change
1536     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1537
1538     # Generate csv tables:
1539     with open(f"{table[u'output-file']}.csv", u"wt") as file_handler:
1540         file_handler.write(u";".join(header) + u"\n")
1541         for test in tbl_lst:
1542             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1543
1544     convert_csv_to_pretty_txt(f"{table[u'output-file']}.csv",
1545                               f"{table[u'output-file']}.txt",
1546                               delimiter=u";")
1547
1548     with open(table[u'output-file'], u'a') as txt_file:
1549         txt_file.write(legend)
1550
1551     # Generate html table:
1552     _tpc_generate_html_table(
1553         header,
1554         tbl_lst,
1555         table[u'output-file'],
1556         legend=legend
1557     )
1558
1559
1560 def table_soak_vs_ndr(table, input_data):
1561     """Generate the table(s) with algorithm: table_soak_vs_ndr
1562     specified in the specification file.
1563
1564     :param table: Table to generate.
1565     :param input_data: Data to process.
1566     :type table: pandas.Series
1567     :type input_data: InputData
1568     """
1569
1570     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1571
1572     # Transform the data
1573     logging.info(
1574         f"    Creating the data set for the {table.get(u'type', u'')} "
1575         f"{table.get(u'title', u'')}."
1576     )
1577     data = input_data.filter_data(table, continue_on_error=True)
1578
1579     # Prepare the header of the table
1580     try:
1581         header = [
1582             u"Test Case",
1583             f"Avg({table[u'reference'][u'title']})",
1584             f"Stdev({table[u'reference'][u'title']})",
1585             f"Avg({table[u'compare'][u'title']})",
1586             f"Stdev{table[u'compare'][u'title']})",
1587             u"Diff",
1588             u"Stdev(Diff)"
1589         ]
1590         header_str = u";".join(header) + u"\n"
1591         legend = (
1592             u"\nLegend:\n"
1593             f"Avg({table[u'reference'][u'title']}): "
1594             f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
1595             f"from a series of runs of the listed tests.\n"
1596             f"Stdev({table[u'reference'][u'title']}): "
1597             f"Standard deviation value of {table[u'reference'][u'title']} "
1598             f"[Mpps] computed from a series of runs of the listed tests.\n"
1599             f"Avg({table[u'compare'][u'title']}): "
1600             f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
1601             f"a series of runs of the listed tests.\n"
1602             f"Stdev({table[u'compare'][u'title']}): "
1603             f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
1604             f"computed from a series of runs of the listed tests.\n"
1605             f"Diff({table[u'reference'][u'title']},"
1606             f"{table[u'compare'][u'title']}): "
1607             f"Percentage change calculated for mean values.\n"
1608             u"Stdev(Diff): "
1609             u"Standard deviation of percentage change calculated for mean "
1610             u"values.\n"
1611             u":END"
1612         )
1613     except (AttributeError, KeyError) as err:
1614         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1615         return
1616
1617     # Create a list of available SOAK test results:
1618     tbl_dict = dict()
1619     for job, builds in table[u"compare"][u"data"].items():
1620         for build in builds:
1621             for tst_name, tst_data in data[job][str(build)].items():
1622                 if tst_data[u"type"] == u"SOAK":
1623                     tst_name_mod = tst_name.replace(u"-soak", u"")
1624                     if tbl_dict.get(tst_name_mod, None) is None:
1625                         groups = re.search(REGEX_NIC, tst_data[u"parent"])
1626                         nic = groups.group(0) if groups else u""
1627                         name = (
1628                             f"{nic}-"
1629                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
1630                         )
1631                         tbl_dict[tst_name_mod] = {
1632                             u"name": name,
1633                             u"ref-data": list(),
1634                             u"cmp-data": list()
1635                         }
1636                     try:
1637                         tbl_dict[tst_name_mod][u"cmp-data"].append(
1638                             tst_data[u"throughput"][u"LOWER"])
1639                     except (KeyError, TypeError):
1640                         pass
1641     tests_lst = tbl_dict.keys()
1642
1643     # Add corresponding NDR test results:
1644     for job, builds in table[u"reference"][u"data"].items():
1645         for build in builds:
1646             for tst_name, tst_data in data[job][str(build)].items():
1647                 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
1648                     replace(u"-mrr", u"")
1649                 if tst_name_mod not in tests_lst:
1650                     continue
1651                 try:
1652                     if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
1653                         continue
1654                     if table[u"include-tests"] == u"MRR":
1655                         result = (tst_data[u"result"][u"receive-rate"],
1656                                   tst_data[u"result"][u"receive-stdev"])
1657                     elif table[u"include-tests"] == u"PDR":
1658                         result = \
1659                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
1660                     elif table[u"include-tests"] == u"NDR":
1661                         result = \
1662                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
1663                     else:
1664                         result = None
1665                     if result is not None:
1666                         tbl_dict[tst_name_mod][u"ref-data"].append(
1667                             result)
1668                 except (KeyError, TypeError):
1669                     continue
1670
1671     tbl_lst = list()
1672     for tst_name in tbl_dict:
1673         item = [tbl_dict[tst_name][u"name"], ]
1674         data_r = tbl_dict[tst_name][u"ref-data"]
1675         if data_r:
1676             if table[u"include-tests"] == u"MRR":
1677                 data_r_mean = data_r[0][0]
1678                 data_r_stdev = data_r[0][1]
1679             else:
1680                 data_r_mean = mean(data_r)
1681                 data_r_stdev = stdev(data_r)
1682             item.append(round(data_r_mean / 1e6, 1))
1683             item.append(round(data_r_stdev / 1e6, 1))
1684         else:
1685             data_r_mean = None
1686             data_r_stdev = None
1687             item.extend([None, None])
1688         data_c = tbl_dict[tst_name][u"cmp-data"]
1689         if data_c:
1690             if table[u"include-tests"] == u"MRR":
1691                 data_c_mean = data_c[0][0]
1692                 data_c_stdev = data_c[0][1]
1693             else:
1694                 data_c_mean = mean(data_c)
1695                 data_c_stdev = stdev(data_c)
1696             item.append(round(data_c_mean / 1e6, 1))
1697             item.append(round(data_c_stdev / 1e6, 1))
1698         else:
1699             data_c_mean = None
1700             data_c_stdev = None
1701             item.extend([None, None])
1702         if data_r_mean is not None and data_c_mean is not None:
1703             delta, d_stdev = relative_change_stdev(
1704                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
1705             try:
1706                 item.append(round(delta))
1707             except ValueError:
1708                 item.append(delta)
1709             try:
1710                 item.append(round(d_stdev))
1711             except ValueError:
1712                 item.append(d_stdev)
1713             tbl_lst.append(item)
1714
1715     # Sort the table according to the relative change
1716     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1717
1718     # Generate csv tables:
1719     csv_file = f"{table[u'output-file']}.csv"
1720     with open(csv_file, u"wt") as file_handler:
1721         file_handler.write(header_str)
1722         for test in tbl_lst:
1723             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1724
1725     convert_csv_to_pretty_txt(
1726         csv_file, f"{table[u'output-file']}.txt", delimiter=u";"
1727     )
1728     with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
1729         txt_file.write(legend)
1730
1731     # Generate html table:
1732     _tpc_generate_html_table(
1733         header,
1734         tbl_lst,
1735         table[u'output-file'],
1736         legend=legend
1737     )
1738
1739
1740 def table_perf_trending_dash(table, input_data):
1741     """Generate the table(s) with algorithm:
1742     table_perf_trending_dash
1743     specified in the specification file.
1744
1745     :param table: Table to generate.
1746     :param input_data: Data to process.
1747     :type table: pandas.Series
1748     :type input_data: InputData
1749     """
1750
1751     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1752
1753     # Transform the data
1754     logging.info(
1755         f"    Creating the data set for the {table.get(u'type', u'')} "
1756         f"{table.get(u'title', u'')}."
1757     )
1758     data = input_data.filter_data(table, continue_on_error=True)
1759
1760     # Prepare the header of the tables
1761     header = [
1762         u"Test Case",
1763         u"Trend [Mpps]",
1764         u"Short-Term Change [%]",
1765         u"Long-Term Change [%]",
1766         u"Regressions [#]",
1767         u"Progressions [#]"
1768     ]
1769     header_str = u",".join(header) + u"\n"
1770
1771     # Prepare data to the table:
1772     tbl_dict = dict()
1773     for job, builds in table[u"data"].items():
1774         for build in builds:
1775             for tst_name, tst_data in data[job][str(build)].items():
1776                 if tst_name.lower() in table.get(u"ignore-list", list()):
1777                     continue
1778                 if tbl_dict.get(tst_name, None) is None:
1779                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1780                     if not groups:
1781                         continue
1782                     nic = groups.group(0)
1783                     tbl_dict[tst_name] = {
1784                         u"name": f"{nic}-{tst_data[u'name']}",
1785                         u"data": OrderedDict()
1786                     }
1787                 try:
1788                     tbl_dict[tst_name][u"data"][str(build)] = \
1789                         tst_data[u"result"][u"receive-rate"]
1790                 except (TypeError, KeyError):
1791                     pass  # No data in output.xml for this test
1792
1793     tbl_lst = list()
1794     for tst_name in tbl_dict:
1795         data_t = tbl_dict[tst_name][u"data"]
1796         if len(data_t) < 2:
1797             continue
1798
1799         classification_lst, avgs = classify_anomalies(data_t)
1800
1801         win_size = min(len(data_t), table[u"window"])
1802         long_win_size = min(len(data_t), table[u"long-trend-window"])
1803
1804         try:
1805             max_long_avg = max(
1806                 [x for x in avgs[-long_win_size:-win_size]
1807                  if not isnan(x)])
1808         except ValueError:
1809             max_long_avg = nan
1810         last_avg = avgs[-1]
1811         avg_week_ago = avgs[max(-win_size, -len(avgs))]
1812
1813         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
1814             rel_change_last = nan
1815         else:
1816             rel_change_last = round(
1817                 ((last_avg - avg_week_ago) / avg_week_ago) * 1e2, 2)
1818
1819         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
1820             rel_change_long = nan
1821         else:
1822             rel_change_long = round(
1823                 ((last_avg - max_long_avg) / max_long_avg) * 1e2, 2)
1824
1825         if classification_lst:
1826             if isnan(rel_change_last) and isnan(rel_change_long):
1827                 continue
1828             if isnan(last_avg) or isnan(rel_change_last) or \
1829                     isnan(rel_change_long):
1830                 continue
1831             tbl_lst.append(
1832                 [tbl_dict[tst_name][u"name"],
1833                  round(last_avg / 1e6, 2),
1834                  rel_change_last,
1835                  rel_change_long,
1836                  classification_lst[-win_size:].count(u"regression"),
1837                  classification_lst[-win_size:].count(u"progression")])
1838
1839     tbl_lst.sort(key=lambda rel: rel[0])
1840
1841     tbl_sorted = list()
1842     for nrr in range(table[u"window"], -1, -1):
1843         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
1844         for nrp in range(table[u"window"], -1, -1):
1845             tbl_out = [item for item in tbl_reg if item[5] == nrp]
1846             tbl_out.sort(key=lambda rel: rel[2])
1847             tbl_sorted.extend(tbl_out)
1848
1849     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1850
1851     logging.info(f"    Writing file: {file_name}")
1852     with open(file_name, u"wt") as file_handler:
1853         file_handler.write(header_str)
1854         for test in tbl_sorted:
1855             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1856
1857     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1858     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1859
1860
1861 def _generate_url(testbed, test_name):
1862     """Generate URL to a trending plot from the name of the test case.
1863
1864     :param testbed: The testbed used for testing.
1865     :param test_name: The name of the test case.
1866     :type testbed: str
1867     :type test_name: str
1868     :returns: The URL to the plot with the trending data for the given test
1869         case.
1870     :rtype str
1871     """
1872
1873     if u"x520" in test_name:
1874         nic = u"x520"
1875     elif u"x710" in test_name:
1876         nic = u"x710"
1877     elif u"xl710" in test_name:
1878         nic = u"xl710"
1879     elif u"xxv710" in test_name:
1880         nic = u"xxv710"
1881     elif u"vic1227" in test_name:
1882         nic = u"vic1227"
1883     elif u"vic1385" in test_name:
1884         nic = u"vic1385"
1885     elif u"x553" in test_name:
1886         nic = u"x553"
1887     elif u"cx556" in test_name or u"cx556a" in test_name:
1888         nic = u"cx556a"
1889     else:
1890         nic = u""
1891
1892     if u"64b" in test_name:
1893         frame_size = u"64b"
1894     elif u"78b" in test_name:
1895         frame_size = u"78b"
1896     elif u"imix" in test_name:
1897         frame_size = u"imix"
1898     elif u"9000b" in test_name:
1899         frame_size = u"9000b"
1900     elif u"1518b" in test_name:
1901         frame_size = u"1518b"
1902     elif u"114b" in test_name:
1903         frame_size = u"114b"
1904     else:
1905         frame_size = u""
1906
1907     if u"1t1c" in test_name or \
1908         (u"-1c-" in test_name and
1909          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1910         cores = u"1t1c"
1911     elif u"2t2c" in test_name or \
1912          (u"-2c-" in test_name and
1913           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1914         cores = u"2t2c"
1915     elif u"4t4c" in test_name or \
1916          (u"-4c-" in test_name and
1917           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1918         cores = u"4t4c"
1919     elif u"2t1c" in test_name or \
1920          (u"-1c-" in test_name and
1921           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1922         cores = u"2t1c"
1923     elif u"4t2c" in test_name or \
1924          (u"-2c-" in test_name and
1925           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1926         cores = u"4t2c"
1927     elif u"8t4c" in test_name or \
1928          (u"-4c-" in test_name and
1929           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1930         cores = u"8t4c"
1931     else:
1932         cores = u""
1933
1934     if u"testpmd" in test_name:
1935         driver = u"testpmd"
1936     elif u"l3fwd" in test_name:
1937         driver = u"l3fwd"
1938     elif u"avf" in test_name:
1939         driver = u"avf"
1940     elif u"rdma" in test_name:
1941         driver = u"rdma"
1942     elif u"dnv" in testbed or u"tsh" in testbed:
1943         driver = u"ixgbe"
1944     else:
1945         driver = u"dpdk"
1946
1947     if u"acl" in test_name or \
1948             u"macip" in test_name or \
1949             u"nat" in test_name or \
1950             u"policer" in test_name or \
1951             u"cop" in test_name:
1952         bsf = u"features"
1953     elif u"scale" in test_name:
1954         bsf = u"scale"
1955     elif u"base" in test_name:
1956         bsf = u"base"
1957     else:
1958         bsf = u"base"
1959
1960     if u"114b" in test_name and u"vhost" in test_name:
1961         domain = u"vts"
1962     elif u"testpmd" in test_name or u"l3fwd" in test_name:
1963         domain = u"dpdk"
1964     elif u"memif" in test_name:
1965         domain = u"container_memif"
1966     elif u"srv6" in test_name:
1967         domain = u"srv6"
1968     elif u"vhost" in test_name:
1969         domain = u"vhost"
1970         if u"vppl2xc" in test_name:
1971             driver += u"-vpp"
1972         else:
1973             driver += u"-testpmd"
1974         if u"lbvpplacp" in test_name:
1975             bsf += u"-link-bonding"
1976     elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1977         domain = u"nf_service_density_vnfc"
1978     elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1979         domain = u"nf_service_density_cnfc"
1980     elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1981         domain = u"nf_service_density_cnfp"
1982     elif u"ipsec" in test_name:
1983         domain = u"ipsec"
1984         if u"sw" in test_name:
1985             bsf += u"-sw"
1986         elif u"hw" in test_name:
1987             bsf += u"-hw"
1988     elif u"ethip4vxlan" in test_name:
1989         domain = u"ip4_tunnels"
1990     elif u"ip4base" in test_name or u"ip4scale" in test_name:
1991         domain = u"ip4"
1992     elif u"ip6base" in test_name or u"ip6scale" in test_name:
1993         domain = u"ip6"
1994     elif u"l2xcbase" in test_name or \
1995             u"l2xcscale" in test_name or \
1996             u"l2bdbasemaclrn" in test_name or \
1997             u"l2bdscale" in test_name or \
1998             u"l2patch" in test_name:
1999         domain = u"l2"
2000     else:
2001         domain = u""
2002
2003     file_name = u"-".join((domain, testbed, nic)) + u".html#"
2004     anchor_name = u"-".join((frame_size, cores, bsf, driver))
2005
2006     return file_name + anchor_name
2007
2008
2009 def table_perf_trending_dash_html(table, input_data):
2010     """Generate the table(s) with algorithm:
2011     table_perf_trending_dash_html specified in the specification
2012     file.
2013
2014     :param table: Table to generate.
2015     :param input_data: Data to process.
2016     :type table: dict
2017     :type input_data: InputData
2018     """
2019
2020     _ = input_data
2021
2022     if not table.get(u"testbed", None):
2023         logging.error(
2024             f"The testbed is not defined for the table "
2025             f"{table.get(u'title', u'')}."
2026         )
2027         return
2028
2029     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
2030
2031     try:
2032         with open(table[u"input-file"], u'rt') as csv_file:
2033             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
2034     except KeyError:
2035         logging.warning(u"The input file is not defined.")
2036         return
2037     except csv.Error as err:
2038         logging.warning(
2039             f"Not possible to process the file {table[u'input-file']}.\n"
2040             f"{repr(err)}"
2041         )
2042         return
2043
2044     # Table:
2045     dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
2046
2047     # Table header:
2048     trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
2049     for idx, item in enumerate(csv_lst[0]):
2050         alignment = u"left" if idx == 0 else u"center"
2051         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
2052         thead.text = item
2053
2054     # Rows:
2055     colors = {
2056         u"regression": (
2057             u"#ffcccc",
2058             u"#ff9999"
2059         ),
2060         u"progression": (
2061             u"#c6ecc6",
2062             u"#9fdf9f"
2063         ),
2064         u"normal": (
2065             u"#e9f1fb",
2066             u"#d4e4f7"
2067         )
2068     }
2069     for r_idx, row in enumerate(csv_lst[1:]):
2070         if int(row[4]):
2071             color = u"regression"
2072         elif int(row[5]):
2073             color = u"progression"
2074         else:
2075             color = u"normal"
2076         trow = ET.SubElement(
2077             dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
2078         )
2079
2080         # Columns:
2081         for c_idx, item in enumerate(row):
2082             tdata = ET.SubElement(
2083                 trow,
2084                 u"td",
2085                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
2086             )
2087             # Name:
2088             if c_idx == 0:
2089                 ref = ET.SubElement(
2090                     tdata,
2091                     u"a",
2092                     attrib=dict(
2093                         href=f"../trending/"
2094                              f"{_generate_url(table.get(u'testbed', ''), item)}"
2095                     )
2096                 )
2097                 ref.text = item
2098             else:
2099                 tdata.text = item
2100     try:
2101         with open(table[u"output-file"], u'w') as html_file:
2102             logging.info(f"    Writing file: {table[u'output-file']}")
2103             html_file.write(u".. raw:: html\n\n\t")
2104             html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
2105             html_file.write(u"\n\t<p><br><br></p>\n")
2106     except KeyError:
2107         logging.warning(u"The output file is not defined.")
2108         return
2109
2110
2111 def table_last_failed_tests(table, input_data):
2112     """Generate the table(s) with algorithm: table_last_failed_tests
2113     specified in the specification file.
2114
2115     :param table: Table to generate.
2116     :param input_data: Data to process.
2117     :type table: pandas.Series
2118     :type input_data: InputData
2119     """
2120
2121     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
2122
2123     # Transform the data
2124     logging.info(
2125         f"    Creating the data set for the {table.get(u'type', u'')} "
2126         f"{table.get(u'title', u'')}."
2127     )
2128
2129     data = input_data.filter_data(table, continue_on_error=True)
2130
2131     if data is None or data.empty:
2132         logging.warning(
2133             f"    No data for the {table.get(u'type', u'')} "
2134             f"{table.get(u'title', u'')}."
2135         )
2136         return
2137
2138     tbl_list = list()
2139     for job, builds in table[u"data"].items():
2140         for build in builds:
2141             build = str(build)
2142             try:
2143                 version = input_data.metadata(job, build).get(u"version", u"")
2144             except KeyError:
2145                 logging.error(f"Data for {job}: {build} is not present.")
2146                 return
2147             tbl_list.append(build)
2148             tbl_list.append(version)
2149             failed_tests = list()
2150             passed = 0
2151             failed = 0
2152             for tst_data in data[job][build].values:
2153                 if tst_data[u"status"] != u"FAIL":
2154                     passed += 1
2155                     continue
2156                 failed += 1
2157                 groups = re.search(REGEX_NIC, tst_data[u"parent"])
2158                 if not groups:
2159                     continue
2160                 nic = groups.group(0)
2161                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
2162             tbl_list.append(str(passed))
2163             tbl_list.append(str(failed))
2164             tbl_list.extend(failed_tests)
2165
2166     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
2167     logging.info(f"    Writing file: {file_name}")
2168     with open(file_name, u"wt") as file_handler:
2169         for test in tbl_list:
2170             file_handler.write(test + u'\n')
2171
2172
2173 def table_failed_tests(table, input_data):
2174     """Generate the table(s) with algorithm: table_failed_tests
2175     specified in the specification file.
2176
2177     :param table: Table to generate.
2178     :param input_data: Data to process.
2179     :type table: pandas.Series
2180     :type input_data: InputData
2181     """
2182
2183     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
2184
2185     # Transform the data
2186     logging.info(
2187         f"    Creating the data set for the {table.get(u'type', u'')} "
2188         f"{table.get(u'title', u'')}."
2189     )
2190     data = input_data.filter_data(table, continue_on_error=True)
2191
2192     # Prepare the header of the tables
2193     header = [
2194         u"Test Case",
2195         u"Failures [#]",
2196         u"Last Failure [Time]",
2197         u"Last Failure [VPP-Build-Id]",
2198         u"Last Failure [CSIT-Job-Build-Id]"
2199     ]
2200
2201     # Generate the data for the table according to the model in the table
2202     # specification
2203
2204     now = dt.utcnow()
2205     timeperiod = timedelta(int(table.get(u"window", 7)))
2206
2207     tbl_dict = dict()
2208     for job, builds in table[u"data"].items():
2209         for build in builds:
2210             build = str(build)
2211             for tst_name, tst_data in data[job][build].items():
2212                 if tst_name.lower() in table.get(u"ignore-list", list()):
2213                     continue
2214                 if tbl_dict.get(tst_name, None) is None:
2215                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
2216                     if not groups:
2217                         continue
2218                     nic = groups.group(0)
2219                     tbl_dict[tst_name] = {
2220                         u"name": f"{nic}-{tst_data[u'name']}",
2221                         u"data": OrderedDict()
2222                     }
2223                 try:
2224                     generated = input_data.metadata(job, build).\
2225                         get(u"generated", u"")
2226                     if not generated:
2227                         continue
2228                     then = dt.strptime(generated, u"%Y%m%d %H:%M")
2229                     if (now - then) <= timeperiod:
2230                         tbl_dict[tst_name][u"data"][build] = (
2231                             tst_data[u"status"],
2232                             generated,
2233                             input_data.metadata(job, build).get(u"version",
2234                                                                 u""),
2235                             build
2236                         )
2237                 except (TypeError, KeyError) as err:
2238                     logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
2239
2240     max_fails = 0
2241     tbl_lst = list()
2242     for tst_data in tbl_dict.values():
2243         fails_nr = 0
2244         fails_last_date = u""
2245         fails_last_vpp = u""
2246         fails_last_csit = u""
2247         for val in tst_data[u"data"].values():
2248             if val[0] == u"FAIL":
2249                 fails_nr += 1
2250                 fails_last_date = val[1]
2251                 fails_last_vpp = val[2]
2252                 fails_last_csit = val[3]
2253         if fails_nr:
2254             max_fails = fails_nr if fails_nr > max_fails else max_fails
2255             tbl_lst.append(
2256                 [
2257                     tst_data[u"name"],
2258                     fails_nr,
2259                     fails_last_date,
2260                     fails_last_vpp,
2261                     f"mrr-daily-build-{fails_last_csit}"
2262                 ]
2263             )
2264
2265     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
2266     tbl_sorted = list()
2267     for nrf in range(max_fails, -1, -1):
2268         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
2269         tbl_sorted.extend(tbl_fails)
2270
2271     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
2272     logging.info(f"    Writing file: {file_name}")
2273     with open(file_name, u"wt") as file_handler:
2274         file_handler.write(u",".join(header) + u"\n")
2275         for test in tbl_sorted:
2276             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
2277
2278     logging.info(f"    Writing file: {table[u'output-file']}.txt")
2279     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
2280
2281
2282 def table_failed_tests_html(table, input_data):
2283     """Generate the table(s) with algorithm: table_failed_tests_html
2284     specified in the specification file.
2285
2286     :param table: Table to generate.
2287     :param input_data: Data to process.
2288     :type table: pandas.Series
2289     :type input_data: InputData
2290     """
2291
2292     _ = input_data
2293
2294     if not table.get(u"testbed", None):
2295         logging.error(
2296             f"The testbed is not defined for the table "
2297             f"{table.get(u'title', u'')}."
2298         )
2299         return
2300
2301     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
2302
2303     try:
2304         with open(table[u"input-file"], u'rt') as csv_file:
2305             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
2306     except KeyError:
2307         logging.warning(u"The input file is not defined.")
2308         return
2309     except csv.Error as err:
2310         logging.warning(
2311             f"Not possible to process the file {table[u'input-file']}.\n"
2312             f"{repr(err)}"
2313         )
2314         return
2315
2316     # Table:
2317     failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
2318
2319     # Table header:
2320     trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
2321     for idx, item in enumerate(csv_lst[0]):
2322         alignment = u"left" if idx == 0 else u"center"
2323         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
2324         thead.text = item
2325
2326     # Rows:
2327     colors = (u"#e9f1fb", u"#d4e4f7")
2328     for r_idx, row in enumerate(csv_lst[1:]):
2329         background = colors[r_idx % 2]
2330         trow = ET.SubElement(
2331             failed_tests, u"tr", attrib=dict(bgcolor=background)
2332         )
2333
2334         # Columns:
2335         for c_idx, item in enumerate(row):
2336             tdata = ET.SubElement(
2337                 trow,
2338                 u"td",
2339                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
2340             )
2341             # Name:
2342             if c_idx == 0:
2343                 ref = ET.SubElement(
2344                     tdata,
2345                     u"a",
2346                     attrib=dict(
2347                         href=f"../trending/"
2348                              f"{_generate_url(table.get(u'testbed', ''), item)}"
2349                     )
2350                 )
2351                 ref.text = item
2352             else:
2353                 tdata.text = item
2354     try:
2355         with open(table[u"output-file"], u'w') as html_file:
2356             logging.info(f"    Writing file: {table[u'output-file']}")
2357             html_file.write(u".. raw:: html\n\n\t")
2358             html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
2359             html_file.write(u"\n\t<p><br><br></p>\n")
2360     except KeyError:
2361         logging.warning(u"The output file is not defined.")
2362         return
2363
2364
2365 def table_comparison(table, input_data):
2366     """Generate the table(s) with algorithm: table_comparison
2367     specified in the specification file.
2368
2369     :param table: Table to generate.
2370     :param input_data: Data to process.
2371     :type table: pandas.Series
2372     :type input_data: InputData
2373     """
2374     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
2375
2376     # Transform the data
2377     logging.info(
2378         f"    Creating the data set for the {table.get(u'type', u'')} "
2379         f"{table.get(u'title', u'')}."
2380     )
2381
2382     columns = table.get(u"columns", None)
2383     if not columns:
2384         logging.error(
2385             f"No columns specified for {table.get(u'title', u'')}. Skipping."
2386         )
2387         return
2388
2389     cols = list()
2390     for idx, col in enumerate(columns):
2391         if col.get(u"data-set", None) is None:
2392             logging.warning(f"No data for column {col.get(u'title', u'')}")
2393             continue
2394         data = input_data.filter_data(
2395             table,
2396             params=[u"throughput", u"result", u"name", u"parent", u"tags"],
2397             data=col[u"data-set"],
2398             continue_on_error=True
2399         )
2400         col_data = {
2401             u"title": col.get(u"title", f"Column{idx}"),
2402             u"data": dict()
2403         }
2404         for builds in data.values:
2405             for build in builds:
2406                 for tst_name, tst_data in build.items():
2407                     tst_name_mod = \
2408                         _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
2409                     if col_data[u"data"].get(tst_name_mod, None) is None:
2410                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
2411                         if u"across testbeds" in table[u"title"].lower() or \
2412                                 u"across topologies" in table[u"title"].lower():
2413                             name = _tpc_modify_displayed_test_name(name)
2414                         col_data[u"data"][tst_name_mod] = {
2415                             u"name": name,
2416                             u"replace": True,
2417                             u"data": list(),
2418                             u"mean": None,
2419                             u"stdev": None
2420                         }
2421                     _tpc_insert_data(
2422                         target=col_data[u"data"][tst_name_mod][u"data"],
2423                         src=tst_data,
2424                         include_tests=table[u"include-tests"]
2425                     )
2426
2427         replacement = col.get(u"data-replacement", None)
2428         if replacement:
2429             rpl_data = input_data.filter_data(
2430                 table,
2431                 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
2432                 data=replacement,
2433                 continue_on_error=True
2434             )
2435             for builds in rpl_data.values:
2436                 for build in builds:
2437                     for tst_name, tst_data in build.items():
2438                         tst_name_mod = \
2439                             _tpc_modify_test_name(tst_name).\
2440                             replace(u"2n1l-", u"")
2441                         if col_data[u"data"].get(tst_name_mod, None) is None:
2442                             name = tst_data[u'name'].rsplit(u'-', 1)[0]
2443                             if u"across testbeds" in table[u"title"].lower() \
2444                                     or u"across topologies" in \
2445                                     table[u"title"].lower():
2446                                 name = _tpc_modify_displayed_test_name(name)
2447                             col_data[u"data"][tst_name_mod] = {
2448                                 u"name": name,
2449                                 u"replace": False,
2450                                 u"data": list(),
2451                                 u"mean": None,
2452                                 u"stdev": None
2453                             }
2454                         if col_data[u"data"][tst_name_mod][u"replace"]:
2455                             col_data[u"data"][tst_name_mod][u"replace"] = False
2456                             col_data[u"data"][tst_name_mod][u"data"] = list()
2457                         _tpc_insert_data(
2458                             target=col_data[u"data"][tst_name_mod][u"data"],
2459                             src=tst_data,
2460                             include_tests=table[u"include-tests"]
2461                         )
2462
2463         if table[u"include-tests"] in (u"NDR", u"PDR"):
2464             for tst_name, tst_data in col_data[u"data"].items():
2465                 if tst_data[u"data"]:
2466                     tst_data[u"mean"] = mean(tst_data[u"data"])
2467                     tst_data[u"stdev"] = stdev(tst_data[u"data"])
2468         elif table[u"include-tests"] in (u"MRR", ):
2469             for tst_name, tst_data in col_data[u"data"].items():
2470                 if tst_data[u"data"]:
2471                     tst_data[u"mean"] = tst_data[u"data"][0]
2472                     tst_data[u"stdev"] = tst_data[u"data"][0]
2473
2474         cols.append(col_data)
2475
2476     tbl_dict = dict()
2477     for col in cols:
2478         for tst_name, tst_data in col[u"data"].items():
2479             if tbl_dict.get(tst_name, None) is None:
2480                 tbl_dict[tst_name] = {
2481                     "name": tst_data[u"name"]
2482                 }
2483             tbl_dict[tst_name][col[u"title"]] = {
2484                 u"mean": tst_data[u"mean"],
2485                 u"stdev": tst_data[u"stdev"]
2486             }
2487
2488     tbl_lst = list()
2489     for tst_data in tbl_dict.values():
2490         row = [tst_data[u"name"], ]
2491         for col in cols:
2492             row.append(tst_data.get(col[u"title"], None))
2493         tbl_lst.append(row)
2494
2495     comparisons = table.get(u"comparisons", None)
2496     if comparisons and isinstance(comparisons, list):
2497         for idx, comp in enumerate(comparisons):
2498             try:
2499                 col_ref = int(comp[u"reference"])
2500                 col_cmp = int(comp[u"compare"])
2501             except KeyError:
2502                 logging.warning(u"Comparison: No references defined! Skipping.")
2503                 comparisons.pop(idx)
2504                 continue
2505             if not (0 < col_ref <= len(cols) and
2506                     0 < col_cmp <= len(cols)) or \
2507                     col_ref == col_cmp:
2508                 logging.warning(f"Wrong values of reference={col_ref} "
2509                                 f"and/or compare={col_cmp}. Skipping.")
2510                 comparisons.pop(idx)
2511                 continue
2512
2513     tbl_cmp_lst = list()
2514     if comparisons:
2515         for row in tbl_lst:
2516             new_row = deepcopy(row)
2517             add_to_tbl = False
2518             for comp in comparisons:
2519                 ref_itm = row[int(comp[u"reference"])]
2520                 if ref_itm is None and \
2521                         comp.get(u"reference-alt", None) is not None:
2522                     ref_itm = row[int(comp[u"reference-alt"])]
2523                 cmp_itm = row[int(comp[u"compare"])]
2524                 if ref_itm is not None and cmp_itm is not None and \
2525                         ref_itm[u"mean"] is not None and \
2526                         cmp_itm[u"mean"] is not None and \
2527                         ref_itm[u"stdev"] is not None and \
2528                         cmp_itm[u"stdev"] is not None:
2529                     delta, d_stdev = relative_change_stdev(
2530                         ref_itm[u"mean"], cmp_itm[u"mean"],
2531                         ref_itm[u"stdev"], cmp_itm[u"stdev"]
2532                     )
2533                     new_row.append(
2534                         {
2535                             u"mean": delta * 1e6,
2536                             u"stdev": d_stdev * 1e6
2537                         }
2538                     )
2539                     add_to_tbl = True
2540                 else:
2541                     new_row.append(None)
2542             if add_to_tbl:
2543                 tbl_cmp_lst.append(new_row)
2544
2545     tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
2546     tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
2547
2548     rcas = list()
2549     rca_in = table.get(u"rca", None)
2550     if rca_in and isinstance(rca_in, list):
2551         for idx, itm in enumerate(rca_in):
2552             try:
2553                 with open(itm.get(u"data", u""), u"r") as rca_file:
2554                     rcas.append(
2555                         {
2556                             u"title": itm.get(u"title", f"RCA{idx}"),
2557                             u"data": load(rca_file, Loader=FullLoader)
2558                         }
2559                     )
2560             except (YAMLError, IOError) as err:
2561                 logging.warning(
2562                     f"The RCA file {itm.get(u'data', u'')} does not exist or "
2563                     f"it is corrupted!"
2564                 )
2565                 logging.debug(repr(err))
2566
2567     tbl_for_csv = list()
2568     for line in tbl_cmp_lst:
2569         row = [line[0], ]
2570         for idx, itm in enumerate(line[1:]):
2571             if itm is None:
2572                 row.append(u"NT")
2573                 row.append(u"NT")
2574             else:
2575                 row.append(round(float(itm[u'mean']) / 1e6, 3))
2576                 row.append(round(float(itm[u'stdev']) / 1e6, 3))
2577         for rca in rcas:
2578             rca_nr = rca[u"data"].get(row[0], u"-")
2579             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
2580         tbl_for_csv.append(row)
2581
2582     header_csv = [u"Test Case", ]
2583     for col in cols:
2584         header_csv.append(f"Avg({col[u'title']})")
2585         header_csv.append(f"Stdev({col[u'title']})")
2586     for comp in comparisons:
2587         header_csv.append(
2588             f"Avg({comp.get(u'title', u'')}"
2589         )
2590         header_csv.append(
2591             f"Stdev({comp.get(u'title', u'')})"
2592         )
2593     header_csv.extend([rca[u"title"] for rca in rcas])
2594
2595     legend_lst = table.get(u"legend", None)
2596     if legend_lst is None:
2597         legend = u""
2598     else:
2599         legend = u"\n" + u"\n".join(legend_lst) + u"\n"
2600
2601     footnote = u""
2602     for rca in rcas:
2603         footnote += f"\n{rca[u'title']}:\n"
2604         footnote += rca[u"data"].get(u"footnote", u"")
2605
2606     csv_file = f"{table[u'output-file']}-csv.csv"
2607     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
2608         file_handler.write(
2609             u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
2610         )
2611         for test in tbl_for_csv:
2612             file_handler.write(
2613                 u",".join([f'"{item}"' for item in test]) + u"\n"
2614             )
2615         if legend_lst:
2616             for item in legend_lst:
2617                 file_handler.write(f'"{item}"\n')
2618         if footnote:
2619             for itm in footnote.split(u"\n"):
2620                 file_handler.write(f'"{itm}"\n')
2621
2622     tbl_tmp = list()
2623     max_lens = [0, ] * len(tbl_cmp_lst[0])
2624     for line in tbl_cmp_lst:
2625         row = [line[0], ]
2626         for idx, itm in enumerate(line[1:]):
2627             if itm is None:
2628                 new_itm = u"NT"
2629             else:
2630                 if idx < len(cols):
2631                     new_itm = (
2632                         f"{round(float(itm[u'mean']) / 1e6, 1)} "
2633                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
2634                         replace(u"nan", u"NaN")
2635                     )
2636                 else:
2637                     new_itm = (
2638                         f"{round(float(itm[u'mean']) / 1e6, 1):+} "
2639                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
2640                         replace(u"nan", u"NaN")
2641                     )
2642             if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]:
2643                 max_lens[idx] = len(new_itm.rsplit(u" ", 1)[-1])
2644             row.append(new_itm)
2645
2646         tbl_tmp.append(row)
2647
2648     tbl_final = list()
2649     for line in tbl_tmp:
2650         row = [line[0], ]
2651         for idx, itm in enumerate(line[1:]):
2652             if itm in (u"NT", u"NaN"):
2653                 row.append(itm)
2654                 continue
2655             itm_lst = itm.rsplit(u"\u00B1", 1)
2656             itm_lst[-1] = \
2657                 f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
2658             row.append(u"\u00B1".join(itm_lst))
2659         for rca in rcas:
2660             rca_nr = rca[u"data"].get(row[0], u"-")
2661             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
2662
2663         tbl_final.append(row)
2664
2665     header = [u"Test Case", ]
2666     header.extend([col[u"title"] for col in cols])
2667     header.extend([comp.get(u"title", u"") for comp in comparisons])
2668     header.extend([rca[u"title"] for rca in rcas])
2669
2670     # Generate csv tables:
2671     csv_file = f"{table[u'output-file']}.csv"
2672     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
2673         file_handler.write(u";".join(header) + u"\n")
2674         for test in tbl_final:
2675             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
2676
2677     # Generate txt table:
2678     txt_file_name = f"{table[u'output-file']}.txt"
2679     convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
2680
2681     with open(txt_file_name, u'a', encoding='utf-8') as txt_file:
2682         txt_file.write(legend)
2683         if footnote:
2684             txt_file.write(footnote)
2685         txt_file.write(u":END")
2686
2687     # Generate html table:
2688     _tpc_generate_html_table(
2689         header,
2690         tbl_final,
2691         table[u'output-file'],
2692         legend=legend,
2693         footnote=footnote,
2694         sort_data=False,
2695         title=table.get(u"title", u"")
2696     )