resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from collections import OrderedDict
  23 from xml.etree import ElementTree as ET
  24 from datetime import datetime as dt
  25 from datetime import timedelta
  26 from copy import deepcopy
  27
  28 import plotly.graph_objects as go
  29 import plotly.offline as ploff
  30 import pandas as pd
  31
  32 from numpy import nan, isnan
  33 from yaml import load, FullLoader, YAMLError
  34
  35 from pal_utils import mean, stdev, classify_anomalies, \
  36     convert_csv_to_pretty_txt, relative_change_stdev
  37
  38
  39 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
  40
  41
  42 def generate_tables(spec, data):
  43     """Generate all tables specified in the specification file.
  44
  45     :param spec: Specification read from the specification file.
  46     :param data: Data to process.
  47     :type spec: Specification
  48     :type data: InputData
  49     """
  50
  51     generator = {
  52         u"table_merged_details": table_merged_details,
  53         u"table_perf_comparison": table_perf_comparison,
  54         u"table_perf_comparison_nic": table_perf_comparison_nic,
  55         u"table_nics_comparison": table_nics_comparison,
  56         u"table_soak_vs_ndr": table_soak_vs_ndr,
  57         u"table_perf_trending_dash": table_perf_trending_dash,
  58         u"table_perf_trending_dash_html": table_perf_trending_dash_html,
  59         u"table_last_failed_tests": table_last_failed_tests,
  60         u"table_failed_tests": table_failed_tests,
  61         u"table_failed_tests_html": table_failed_tests_html,
  62         u"table_oper_data_html": table_oper_data_html,
  63         u"table_comparison": table_comparison
  64     }
  65
  66     logging.info(u"Generating the tables ...")
  67     for table in spec.tables:
  68         try:
  69             generator[table[u"algorithm"]](table, data)
  70         except NameError as err:
  71             logging.error(
  72                 f"Probably algorithm {table[u'algorithm']} is not defined: "
  73                 f"{repr(err)}"
  74             )
  75     logging.info(u"Done.")
  76
  77
  78 def table_oper_data_html(table, input_data):
  79     """Generate the table(s) with algorithm: html_table_oper_data
  80     specified in the specification file.
  81
  82     :param table: Table to generate.
  83     :param input_data: Data to process.
  84     :type table: pandas.Series
  85     :type input_data: InputData
  86     """
  87
  88     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
  89     # Transform the data
  90     logging.info(
  91         f"    Creating the data set for the {table.get(u'type', u'')} "
  92         f"{table.get(u'title', u'')}."
  93     )
  94     data = input_data.filter_data(
  95         table,
  96         params=[u"name", u"parent", u"show-run", u"type"],
  97         continue_on_error=True
  98     )
  99     if data.empty:
 100         return
 101     data = input_data.merge_data(data)
 102
 103     sort_tests = table.get(u"sort", None)
 104     if sort_tests:
 105         args = dict(
 106             inplace=True,
 107             ascending=(sort_tests == u"ascending")
 108         )
 109         data.sort_index(**args)
 110
 111     suites = input_data.filter_data(
 112         table,
 113         continue_on_error=True,
 114         data_set=u"suites"
 115     )
 116     if suites.empty:
 117         return
 118     suites = input_data.merge_data(suites)
 119
 120     def _generate_html_table(tst_data):
 121         """Generate an HTML table with operational data for the given test.
 122
 123         :param tst_data: Test data to be used to generate the table.
 124         :type tst_data: pandas.Series
 125         :returns: HTML table with operational data.
 126         :rtype: str
 127         """
 128
 129         colors = {
 130             u"header": u"#7eade7",
 131             u"empty": u"#ffffff",
 132             u"body": (u"#e9f1fb", u"#d4e4f7")
 133         }
 134
 135         tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
 136
 137         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
 138         thead = ET.SubElement(
 139             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 140         )
 141         thead.text = tst_data[u"name"]
 142
 143         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 144         thead = ET.SubElement(
 145             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 146         )
 147         thead.text = u"\t"
 148
 149         if tst_data.get(u"show-run", u"No Data") == u"No Data":
 150             trow = ET.SubElement(
 151                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 152             )
 153             tcol = ET.SubElement(
 154                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 155             )
 156             tcol.text = u"No Data"
 157
 158             trow = ET.SubElement(
 159                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 160             )
 161             thead = ET.SubElement(
 162                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 163             )
 164             font = ET.SubElement(
 165                 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 166             )
 167             font.text = u"."
 168             return str(ET.tostring(tbl, encoding=u"unicode"))
 169
 170         tbl_hdr = (
 171             u"Name",
 172             u"Nr of Vectors",
 173             u"Nr of Packets",
 174             u"Suspends",
 175             u"Cycles per Packet",
 176             u"Average Vector Size"
 177         )
 178
 179         for dut_data in tst_data[u"show-run"].values():
 180             trow = ET.SubElement(
 181                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 182             )
 183             tcol = ET.SubElement(
 184                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 185             )
 186             if dut_data.get(u"threads", None) is None:
 187                 tcol.text = u"No Data"
 188                 continue
 189
 190             bold = ET.SubElement(tcol, u"b")
 191             bold.text = (
 192                 f"Host IP: {dut_data.get(u'host', '')}, "
 193                 f"Socket: {dut_data.get(u'socket', '')}"
 194             )
 195             trow = ET.SubElement(
 196                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 197             )
 198             thead = ET.SubElement(
 199                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 200             )
 201             thead.text = u"\t"
 202
 203             for thread_nr, thread in dut_data[u"threads"].items():
 204                 trow = ET.SubElement(
 205                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 206                 )
 207                 tcol = ET.SubElement(
 208                     trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 209                 )
 210                 bold = ET.SubElement(tcol, u"b")
 211                 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
 212                 trow = ET.SubElement(
 213                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 214                 )
 215                 for idx, col in enumerate(tbl_hdr):
 216                     tcol = ET.SubElement(
 217                         trow, u"td",
 218                         attrib=dict(align=u"right" if idx else u"left")
 219                     )
 220                     font = ET.SubElement(
 221                         tcol, u"font", attrib=dict(size=u"2")
 222                     )
 223                     bold = ET.SubElement(font, u"b")
 224                     bold.text = col
 225                 for row_nr, row in enumerate(thread):
 226                     trow = ET.SubElement(
 227                         tbl, u"tr",
 228                         attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
 229                     )
 230                     for idx, col in enumerate(row):
 231                         tcol = ET.SubElement(
 232                             trow, u"td",
 233                             attrib=dict(align=u"right" if idx else u"left")
 234                         )
 235                         font = ET.SubElement(
 236                             tcol, u"font", attrib=dict(size=u"2")
 237                         )
 238                         if isinstance(col, float):
 239                             font.text = f"{col:.2f}"
 240                         else:
 241                             font.text = str(col)
 242                 trow = ET.SubElement(
 243                     tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 244                 )
 245                 thead = ET.SubElement(
 246                     trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 247                 )
 248                 thead.text = u"\t"
 249
 250         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 251         thead = ET.SubElement(
 252             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 253         )
 254         font = ET.SubElement(
 255             thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 256         )
 257         font.text = u"."
 258
 259         return str(ET.tostring(tbl, encoding=u"unicode"))
 260
 261     for suite in suites.values:
 262         html_table = str()
 263         for test_data in data.values:
 264             if test_data[u"parent"] not in suite[u"name"]:
 265                 continue
 266             html_table += _generate_html_table(test_data)
 267         if not html_table:
 268             continue
 269         try:
 270             file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
 271             with open(f"{file_name}", u'w') as html_file:
 272                 logging.info(f"    Writing file: {file_name}")
 273                 html_file.write(u".. raw:: html\n\n\t")
 274                 html_file.write(html_table)
 275                 html_file.write(u"\n\t<p><br><br></p>\n")
 276         except KeyError:
 277             logging.warning(u"The output file is not defined.")
 278             return
 279     logging.info(u"  Done.")
 280
 281
 282 def table_merged_details(table, input_data):
 283     """Generate the table(s) with algorithm: table_merged_details
 284     specified in the specification file.
 285
 286     :param table: Table to generate.
 287     :param input_data: Data to process.
 288     :type table: pandas.Series
 289     :type input_data: InputData
 290     """
 291
 292     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 293
 294     # Transform the data
 295     logging.info(
 296         f"    Creating the data set for the {table.get(u'type', u'')} "
 297         f"{table.get(u'title', u'')}."
 298     )
 299     data = input_data.filter_data(table, continue_on_error=True)
 300     data = input_data.merge_data(data)
 301
 302     sort_tests = table.get(u"sort", None)
 303     if sort_tests:
 304         args = dict(
 305             inplace=True,
 306             ascending=(sort_tests == u"ascending")
 307         )
 308         data.sort_index(**args)
 309
 310     suites = input_data.filter_data(
 311         table, continue_on_error=True, data_set=u"suites")
 312     suites = input_data.merge_data(suites)
 313
 314     # Prepare the header of the tables
 315     header = list()
 316     for column in table[u"columns"]:
 317         header.append(
 318             u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
 319         )
 320
 321     for suite in suites.values:
 322         # Generate data
 323         suite_name = suite[u"name"]
 324         table_lst = list()
 325         for test in data.keys():
 326             if data[test][u"parent"] not in suite_name:
 327                 continue
 328             row_lst = list()
 329             for column in table[u"columns"]:
 330                 try:
 331                     col_data = str(data[test][column[
 332                         u"data"].split(u" ")[1]]).replace(u'"', u'""')
 333                     # Do not include tests with "Test Failed" in test message
 334                     if u"Test Failed" in col_data:
 335                         continue
 336                     col_data = col_data.replace(
 337                         u"No Data", u"Not Captured     "
 338                     )
 339                     if column[u"data"].split(u" ")[1] in (u"name", ):
 340                         if len(col_data) > 30:
 341                             col_data_lst = col_data.split(u"-")
 342                             half = int(len(col_data_lst) / 2)
 343                             col_data = f"{u'-'.join(col_data_lst[:half])}" \
 344                                        f"- |br| " \
 345                                        f"{u'-'.join(col_data_lst[half:])}"
 346                         col_data = f" |prein| {col_data} |preout| "
 347                     elif column[u"data"].split(u" ")[1] in (u"msg", ):
 348                         # Temporary solution: remove NDR results from message:
 349                         if bool(table.get(u'remove-ndr', False)):
 350                             try:
 351                                 col_data = col_data.split(u" |br| ", 1)[1]
 352                             except IndexError:
 353                                 pass
 354                         col_data = f" |prein| {col_data} |preout| "
 355                     elif column[u"data"].split(u" ")[1] in \
 356                             (u"conf-history", u"show-run"):
 357                         col_data = col_data.replace(u" |br| ", u"", 1)
 358                         col_data = f" |prein| {col_data[:-5]} |preout| "
 359                     row_lst.append(f'"{col_data}"')
 360                 except KeyError:
 361                     row_lst.append(u'"Not captured"')
 362             if len(row_lst) == len(table[u"columns"]):
 363                 table_lst.append(row_lst)
 364
 365         # Write the data to file
 366         if table_lst:
 367             separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
 368             file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
 369             logging.info(f"      Writing file: {file_name}")
 370             with open(file_name, u"wt") as file_handler:
 371                 file_handler.write(u",".join(header) + u"\n")
 372                 for item in table_lst:
 373                     file_handler.write(u",".join(item) + u"\n")
 374
 375     logging.info(u"  Done.")
 376
 377
 378 def _tpc_modify_test_name(test_name, ignore_nic=False):
 379     """Modify a test name by replacing its parts.
 380
 381     :param test_name: Test name to be modified.
 382     :param ignore_nic: If True, NIC is removed from TC name.
 383     :type test_name: str
 384     :type ignore_nic: bool
 385     :returns: Modified test name.
 386     :rtype: str
 387     """
 388     test_name_mod = test_name.\
 389         replace(u"-ndrpdrdisc", u""). \
 390         replace(u"-ndrpdr", u"").\
 391         replace(u"-pdrdisc", u""). \
 392         replace(u"-ndrdisc", u"").\
 393         replace(u"-pdr", u""). \
 394         replace(u"-ndr", u""). \
 395         replace(u"1t1c", u"1c").\
 396         replace(u"2t1c", u"1c"). \
 397         replace(u"2t2c", u"2c").\
 398         replace(u"4t2c", u"2c"). \
 399         replace(u"4t4c", u"4c").\
 400         replace(u"8t4c", u"4c")
 401
 402     if ignore_nic:
 403         return re.sub(REGEX_NIC, u"", test_name_mod)
 404     return test_name_mod
 405
 406
 407 def _tpc_modify_displayed_test_name(test_name):
 408     """Modify a test name which is displayed in a table by replacing its parts.
 409
 410     :param test_name: Test name to be modified.
 411     :type test_name: str
 412     :returns: Modified test name.
 413     :rtype: str
 414     """
 415     return test_name.\
 416         replace(u"1t1c", u"1c").\
 417         replace(u"2t1c", u"1c"). \
 418         replace(u"2t2c", u"2c").\
 419         replace(u"4t2c", u"2c"). \
 420         replace(u"4t4c", u"4c").\
 421         replace(u"8t4c", u"4c")
 422
 423
 424 def _tpc_insert_data(target, src, include_tests):
 425     """Insert src data to the target structure.
 426
 427     :param target: Target structure where the data is placed.
 428     :param src: Source data to be placed into the target stucture.
 429     :param include_tests: Which results will be included (MRR, NDR, PDR).
 430     :type target: list
 431     :type src: dict
 432     :type include_tests: str
 433     """
 434     try:
 435         if include_tests == u"MRR":
 436             target.append(
 437                 (
 438                     src[u"result"][u"receive-rate"],
 439                     src[u"result"][u"receive-stdev"]
 440                 )
 441             )
 442         elif include_tests == u"PDR":
 443             target.append(src[u"throughput"][u"PDR"][u"LOWER"])
 444         elif include_tests == u"NDR":
 445             target.append(src[u"throughput"][u"NDR"][u"LOWER"])
 446     except (KeyError, TypeError):
 447         pass
 448
 449
 450 def _tpc_sort_table(table):
 451     """Sort the table this way:
 452
 453     1. Put "New in CSIT-XXXX" at the first place.
 454     2. Put "See footnote" at the second place.
 455     3. Sort the rest by "Delta".
 456
 457     :param table: Table to sort.
 458     :type table: list
 459     :returns: Sorted table.
 460     :rtype: list
 461     """
 462
 463     tbl_new = list()
 464     tbl_see = list()
 465     tbl_delta = list()
 466     for item in table:
 467         if isinstance(item[-1], str):
 468             if u"New in CSIT" in item[-1]:
 469                 tbl_new.append(item)
 470             elif u"See footnote" in item[-1]:
 471                 tbl_see.append(item)
 472         else:
 473             tbl_delta.append(item)
 474
 475     # Sort the tables:
 476     tbl_new.sort(key=lambda rel: rel[0], reverse=False)
 477     tbl_see.sort(key=lambda rel: rel[0], reverse=False)
 478     tbl_see.sort(key=lambda rel: rel[-2], reverse=False)
 479     tbl_delta.sort(key=lambda rel: rel[0], reverse=False)
 480     tbl_delta.sort(key=lambda rel: rel[-2], reverse=True)
 481
 482     # Put the tables together:
 483     table = list()
 484     # We do not want "New in CSIT":
 485     # table.extend(tbl_new)
 486     table.extend(tbl_see)
 487     table.extend(tbl_delta)
 488
 489     return table
 490
 491
 492 def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
 493                              footnote=u"", sort_data=True, title=u""):
 494     """Generate html table from input data with simple sorting possibility.
 495
 496     :param header: Table header.
 497     :param data: Input data to be included in the table. It is a list of lists.
 498         Inner lists are rows in the table. All inner lists must be of the same
 499         length. The length of these lists must be the same as the length of the
 500         header.
 501     :param out_file_name: The name (relative or full path) where the
 502         generated html table is written.
 503     :param legend: The legend to display below the table.
 504     :param footnote: The footnote to display below the table (and legend).
 505     :param sort_data: If True the data sorting is enabled.
 506     :param title: The table (and file) title.
 507     :type header: list
 508     :type data: list of lists
 509     :type out_file_name: str
 510     :type legend: str
 511     :type footnote: str
 512     :type sort_data: bool
 513     :type title: str
 514     """
 515
 516     try:
 517         idx = header.index(u"Test Case")
 518     except ValueError:
 519         idx = 0
 520     params = {
 521         u"align-hdr": (
 522             [u"left", u"right"],
 523             [u"left", u"left", u"right"],
 524             [u"left", u"left", u"left", u"right"]
 525         ),
 526         u"align-itm": (
 527             [u"left", u"right"],
 528             [u"left", u"left", u"right"],
 529             [u"left", u"left", u"left", u"right"]
 530         ),
 531         u"width": ([28, 9], [4, 24, 10], [4, 4, 32, 10])
 532     }
 533
 534     df_data = pd.DataFrame(data, columns=header)
 535
 536     if sort_data:
 537         df_sorted = [df_data.sort_values(
 538             by=[key, header[idx]], ascending=[True, True]
 539             if key != header[idx] else [False, True]) for key in header]
 540         df_sorted_rev = [df_data.sort_values(
 541             by=[key, header[idx]], ascending=[False, True]
 542             if key != header[idx] else [True, True]) for key in header]
 543         df_sorted.extend(df_sorted_rev)
 544     else:
 545         df_sorted = df_data
 546
 547     fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
 548                    for idx in range(len(df_data))]]
 549     table_header = dict(
 550         values=[f"<b>{item.replace(u',', u',<br>')}</b>" for item in header],
 551         fill_color=u"#7eade7",
 552         align=params[u"align-hdr"][idx],
 553         font=dict(
 554             family=u"Courier New",
 555             size=13
 556         )
 557     )
 558
 559     fig = go.Figure()
 560
 561     if sort_data:
 562         for table in df_sorted:
 563             columns = [table.get(col) for col in header]
 564             fig.add_trace(
 565                 go.Table(
 566                     columnwidth=params[u"width"][idx],
 567                     header=table_header,
 568                     cells=dict(
 569                         values=columns,
 570                         fill_color=fill_color,
 571                         align=params[u"align-itm"][idx],
 572                         font=dict(
 573                             family=u"Courier New",
 574                             size=13
 575                         )
 576                     )
 577                 )
 578             )
 579
 580         buttons = list()
 581         menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
 582         menu_items_rev = [f"<b>{itm}</b> (descending)" for itm in header]
 583         menu_items.extend(menu_items_rev)
 584         for idx, hdr in enumerate(menu_items):
 585             visible = [False, ] * len(menu_items)
 586             visible[idx] = True
 587             buttons.append(
 588                 dict(
 589                     label=hdr.replace(u" [Mpps]", u""),
 590                     method=u"update",
 591                     args=[{u"visible": visible}],
 592                 )
 593             )
 594
 595         fig.update_layout(
 596             updatemenus=[
 597                 go.layout.Updatemenu(
 598                     type=u"dropdown",
 599                     direction=u"down",
 600                     x=0.0,
 601                     xanchor=u"left",
 602                     y=1.045,
 603                     yanchor=u"top",
 604                     active=len(menu_items) - 1,
 605                     buttons=list(buttons)
 606                 )
 607             ],
 608         )
 609     else:
 610         fig.add_trace(
 611             go.Table(
 612                 columnwidth=params[u"width"][idx],
 613                 header=table_header,
 614                 cells=dict(
 615                     values=[df_sorted.get(col) for col in header],
 616                     fill_color=fill_color,
 617                     align=params[u"align-itm"][idx],
 618                     font=dict(
 619                         family=u"Courier New",
 620                         size=13
 621                     )
 622                 )
 623             )
 624         )
 625
 626     ploff.plot(
 627         fig,
 628         show_link=False,
 629         auto_open=False,
 630         filename=f"{out_file_name}_in.html"
 631     )
 632
 633     file_name = out_file_name.split(u"/")[-1]
 634     if u"vpp" in out_file_name:
 635         path = u"_tmp/src/vpp_performance_tests/comparisons/"
 636     else:
 637         path = u"_tmp/src/dpdk_performance_tests/comparisons/"
 638     with open(f"{path}{file_name}.rst", u"wt") as rst_file:
 639         rst_file.write(
 640             u"\n"
 641             u".. |br| raw:: html\n\n    <br />\n\n\n"
 642             u".. |prein| raw:: html\n\n    <pre>\n\n\n"
 643             u".. |preout| raw:: html\n\n    </pre>\n\n"
 644         )
 645         if title:
 646             rst_file.write(f"{title}\n")
 647             rst_file.write(f"{u'~' * len(title)}\n\n")
 648         rst_file.write(
 649             u".. raw:: html\n\n"
 650             f'    <iframe frameborder="0" scrolling="no" '
 651             f'width="1600" height="1200" '
 652             f'src="../..{out_file_name.replace(u"_build", u"")}_in.html">'
 653             f'</iframe>\n\n'
 654         )
 655         if legend:
 656             rst_file.write(legend[1:].replace(u"\n", u" |br| "))
 657         if footnote:
 658             rst_file.write(footnote.replace(u"\n", u" |br| ")[1:])
 659
 660
 661 def table_perf_comparison(table, input_data):
 662     """Generate the table(s) with algorithm: table_perf_comparison
 663     specified in the specification file.
 664
 665     :param table: Table to generate.
 666     :param input_data: Data to process.
 667     :type table: pandas.Series
 668     :type input_data: InputData
 669     """
 670
 671     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 672
 673     # Transform the data
 674     logging.info(
 675         f"    Creating the data set for the {table.get(u'type', u'')} "
 676         f"{table.get(u'title', u'')}."
 677     )
 678     data = input_data.filter_data(table, continue_on_error=True)
 679
 680     # Prepare the header of the tables
 681     try:
 682         header = [u"Test Case", ]
 683         legend = u"\nLegend:\n"
 684
 685         rca_data = None
 686         rca = table.get(u"rca", None)
 687         if rca:
 688             try:
 689                 with open(rca.get(u"data-file", u""), u"r") as rca_file:
 690                     rca_data = load(rca_file, Loader=FullLoader)
 691                 header.insert(0, rca.get(u"title", u"RCA"))
 692                 legend += (
 693                     u"RCA: Reference to the Root Cause Analysis, see below.\n"
 694                 )
 695             except (YAMLError, IOError) as err:
 696                 logging.warning(repr(err))
 697
 698         history = table.get(u"history", list())
 699         for item in history:
 700             header.extend(
 701                 [
 702                     f"{item[u'title']} Avg({table[u'include-tests']})",
 703                     f"{item[u'title']} Stdev({table[u'include-tests']})"
 704                 ]
 705             )
 706             legend += (
 707                 f"{item[u'title']} Avg({table[u'include-tests']}): "
 708                 f"Mean value of {table[u'include-tests']} [Mpps] computed from "
 709                 f"a series of runs of the listed tests executed against "
 710                 f"{item[u'title']}.\n"
 711                 f"{item[u'title']} Stdev({table[u'include-tests']}): "
 712                 f"Standard deviation value of {table[u'include-tests']} [Mpps] "
 713                 f"computed from a series of runs of the listed tests executed "
 714                 f"against {item[u'title']}.\n"
 715             )
 716         header.extend(
 717             [
 718                 f"{table[u'reference'][u'title']} "
 719                 f"Avg({table[u'include-tests']})",
 720                 f"{table[u'reference'][u'title']} "
 721                 f"Stdev({table[u'include-tests']})",
 722                 f"{table[u'compare'][u'title']} "
 723                 f"Avg({table[u'include-tests']})",
 724                 f"{table[u'compare'][u'title']} "
 725                 f"Stdev({table[u'include-tests']})",
 726                 f"Diff({table[u'reference'][u'title']},"
 727                 f"{table[u'compare'][u'title']})",
 728                 u"Stdev(Diff)"
 729             ]
 730         )
 731         header_str = u";".join(header) + u"\n"
 732         legend += (
 733             f"{table[u'reference'][u'title']} "
 734             f"Avg({table[u'include-tests']}): "
 735             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
 736             f"series of runs of the listed tests executed against "
 737             f"{table[u'reference'][u'title']}.\n"
 738             f"{table[u'reference'][u'title']} "
 739             f"Stdev({table[u'include-tests']}): "
 740             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
 741             f"computed from a series of runs of the listed tests executed "
 742             f"against {table[u'reference'][u'title']}.\n"
 743             f"{table[u'compare'][u'title']} "
 744             f"Avg({table[u'include-tests']}): "
 745             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
 746             f"series of runs of the listed tests executed against "
 747             f"{table[u'compare'][u'title']}.\n"
 748             f"{table[u'compare'][u'title']} "
 749             f"Stdev({table[u'include-tests']}): "
 750             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
 751             f"computed from a series of runs of the listed tests executed "
 752             f"against {table[u'compare'][u'title']}.\n"
 753             f"Diff({table[u'reference'][u'title']},"
 754             f"{table[u'compare'][u'title']}): "
 755             f"Percentage change calculated for mean values.\n"
 756             u"Stdev(Diff): "
 757             u"Standard deviation of percentage change calculated for mean "
 758             u"values.\n"
 759             u"NT: Not Tested\n"
 760         )
 761     except (AttributeError, KeyError) as err:
 762         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 763         return
 764
 765     # Prepare data to the table:
 766     tbl_dict = dict()
 767     for job, builds in table[u"reference"][u"data"].items():
 768         for build in builds:
 769             for tst_name, tst_data in data[job][str(build)].items():
 770                 tst_name_mod = _tpc_modify_test_name(tst_name)
 771                 if (u"across topologies" in table[u"title"].lower() or
 772                         (u" 3n-" in table[u"title"].lower() and
 773                          u" 2n-" in table[u"title"].lower())):
 774                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 775                 if tbl_dict.get(tst_name_mod, None) is None:
 776                     name = tst_data[u'name'].rsplit(u'-', 1)[0]
 777                     if u"across testbeds" in table[u"title"].lower() or \
 778                             u"across topologies" in table[u"title"].lower():
 779                         name = _tpc_modify_displayed_test_name(name)
 780                     tbl_dict[tst_name_mod] = {
 781                         u"name": name,
 782                         u"replace-ref": True,
 783                         u"replace-cmp": True,
 784                         u"ref-data": list(),
 785                         u"cmp-data": list()
 786                     }
 787                 _tpc_insert_data(target=tbl_dict[tst_name_mod][u"ref-data"],
 788                                  src=tst_data,
 789                                  include_tests=table[u"include-tests"])
 790
 791     replacement = table[u"reference"].get(u"data-replacement", None)
 792     if replacement:
 793         rpl_data = input_data.filter_data(
 794             table, data=replacement, continue_on_error=True)
 795         for job, builds in replacement.items():
 796             for build in builds:
 797                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 798                     tst_name_mod = _tpc_modify_test_name(tst_name)
 799                     if (u"across topologies" in table[u"title"].lower() or
 800                             (u" 3n-" in table[u"title"].lower() and
 801                              u" 2n-" in table[u"title"].lower())):
 802                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 803                     if tbl_dict.get(tst_name_mod, None) is None:
 804                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
 805                         if u"across testbeds" in table[u"title"].lower() or \
 806                                 u"across topologies" in table[u"title"].lower():
 807                             name = _tpc_modify_displayed_test_name(name)
 808                         tbl_dict[tst_name_mod] = {
 809                             u"name": name,
 810                             u"replace-ref": False,
 811                             u"replace-cmp": True,
 812                             u"ref-data": list(),
 813                             u"cmp-data": list()
 814                         }
 815                     if tbl_dict[tst_name_mod][u"replace-ref"]:
 816                         tbl_dict[tst_name_mod][u"replace-ref"] = False
 817                         tbl_dict[tst_name_mod][u"ref-data"] = list()
 818
 819                     _tpc_insert_data(
 820                         target=tbl_dict[tst_name_mod][u"ref-data"],
 821                         src=tst_data,
 822                         include_tests=table[u"include-tests"]
 823                     )
 824
 825     for job, builds in table[u"compare"][u"data"].items():
 826         for build in builds:
 827             for tst_name, tst_data in data[job][str(build)].items():
 828                 tst_name_mod = _tpc_modify_test_name(tst_name)
 829                 if (u"across topologies" in table[u"title"].lower() or
 830                         (u" 3n-" in table[u"title"].lower() and
 831                          u" 2n-" in table[u"title"].lower())):
 832                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 833                 if tbl_dict.get(tst_name_mod, None) is None:
 834                     name = tst_data[u'name'].rsplit(u'-', 1)[0]
 835                     if u"across testbeds" in table[u"title"].lower() or \
 836                             u"across topologies" in table[u"title"].lower():
 837                         name = _tpc_modify_displayed_test_name(name)
 838                     tbl_dict[tst_name_mod] = {
 839                         u"name": name,
 840                         u"replace-ref": False,
 841                         u"replace-cmp": True,
 842                         u"ref-data": list(),
 843                         u"cmp-data": list()
 844                     }
 845                 _tpc_insert_data(
 846                     target=tbl_dict[tst_name_mod][u"cmp-data"],
 847                     src=tst_data,
 848                     include_tests=table[u"include-tests"]
 849                 )
 850
 851     replacement = table[u"compare"].get(u"data-replacement", None)
 852     if replacement:
 853         rpl_data = input_data.filter_data(
 854             table, data=replacement, continue_on_error=True)
 855         for job, builds in replacement.items():
 856             for build in builds:
 857                 for tst_name, tst_data in rpl_data[job][str(build)].items():
 858                     tst_name_mod = _tpc_modify_test_name(tst_name)
 859                     if (u"across topologies" in table[u"title"].lower() or
 860                             (u" 3n-" in table[u"title"].lower() and
 861                              u" 2n-" in table[u"title"].lower())):
 862                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 863                     if tbl_dict.get(tst_name_mod, None) is None:
 864                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
 865                         if u"across testbeds" in table[u"title"].lower() or \
 866                                 u"across topologies" in table[u"title"].lower():
 867                             name = _tpc_modify_displayed_test_name(name)
 868                         tbl_dict[tst_name_mod] = {
 869                             u"name": name,
 870                             u"replace-ref": False,
 871                             u"replace-cmp": False,
 872                             u"ref-data": list(),
 873                             u"cmp-data": list()
 874                         }
 875                     if tbl_dict[tst_name_mod][u"replace-cmp"]:
 876                         tbl_dict[tst_name_mod][u"replace-cmp"] = False
 877                         tbl_dict[tst_name_mod][u"cmp-data"] = list()
 878
 879                     _tpc_insert_data(
 880                         target=tbl_dict[tst_name_mod][u"cmp-data"],
 881                         src=tst_data,
 882                         include_tests=table[u"include-tests"]
 883                     )
 884
 885     for item in history:
 886         for job, builds in item[u"data"].items():
 887             for build in builds:
 888                 for tst_name, tst_data in data[job][str(build)].items():
 889                     tst_name_mod = _tpc_modify_test_name(tst_name)
 890                     if (u"across topologies" in table[u"title"].lower() or
 891                             (u" 3n-" in table[u"title"].lower() and
 892                              u" 2n-" in table[u"title"].lower())):
 893                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
 894                     if tbl_dict.get(tst_name_mod, None) is None:
 895                         continue
 896                     if tbl_dict[tst_name_mod].get(u"history", None) is None:
 897                         tbl_dict[tst_name_mod][u"history"] = OrderedDict()
 898                     if tbl_dict[tst_name_mod][u"history"].\
 899                             get(item[u"title"], None) is None:
 900                         tbl_dict[tst_name_mod][u"history"][item[
 901                             u"title"]] = list()
 902                     try:
 903                         if table[u"include-tests"] == u"MRR":
 904                             res = (tst_data[u"result"][u"receive-rate"],
 905                                    tst_data[u"result"][u"receive-stdev"])
 906                         elif table[u"include-tests"] == u"PDR":
 907                             res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
 908                         elif table[u"include-tests"] == u"NDR":
 909                             res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
 910                         else:
 911                             continue
 912                         tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
 913                             append(res)
 914                     except (TypeError, KeyError):
 915                         pass
 916
 917     tbl_lst = list()
 918     for tst_name in tbl_dict:
 919         item = [tbl_dict[tst_name][u"name"], ]
 920         if history:
 921             if tbl_dict[tst_name].get(u"history", None) is not None:
 922                 for hist_data in tbl_dict[tst_name][u"history"].values():
 923                     if hist_data:
 924                         if table[u"include-tests"] == u"MRR":
 925                             item.append(round(hist_data[0][0] / 1e6, 1))
 926                             item.append(round(hist_data[0][1] / 1e6, 1))
 927                         else:
 928                             item.append(round(mean(hist_data) / 1e6, 1))
 929                             item.append(round(stdev(hist_data) / 1e6, 1))
 930                     else:
 931                         item.extend([u"NT", u"NT"])
 932             else:
 933                 item.extend([u"NT", u"NT"])
 934         data_r = tbl_dict[tst_name][u"ref-data"]
 935         if data_r:
 936             if table[u"include-tests"] == u"MRR":
 937                 data_r_mean = data_r[0][0]
 938                 data_r_stdev = data_r[0][1]
 939             else:
 940                 data_r_mean = mean(data_r)
 941                 data_r_stdev = stdev(data_r)
 942             item.append(round(data_r_mean / 1e6, 1))
 943             item.append(round(data_r_stdev / 1e6, 1))
 944         else:
 945             data_r_mean = None
 946             data_r_stdev = None
 947             item.extend([u"NT", u"NT"])
 948         data_c = tbl_dict[tst_name][u"cmp-data"]
 949         if data_c:
 950             if table[u"include-tests"] == u"MRR":
 951                 data_c_mean = data_c[0][0]
 952                 data_c_stdev = data_c[0][1]
 953             else:
 954                 data_c_mean = mean(data_c)
 955                 data_c_stdev = stdev(data_c)
 956             item.append(round(data_c_mean / 1e6, 1))
 957             item.append(round(data_c_stdev / 1e6, 1))
 958         else:
 959             data_c_mean = None
 960             data_c_stdev = None
 961             item.extend([u"NT", u"NT"])
 962         if item[-2] == u"NT":
 963             pass
 964         elif item[-4] == u"NT":
 965             item.append(u"New in CSIT-2001")
 966             item.append(u"New in CSIT-2001")
 967         elif data_r_mean is not None and data_c_mean is not None:
 968             delta, d_stdev = relative_change_stdev(
 969                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
 970             )
 971             try:
 972                 item.append(round(delta))
 973             except ValueError:
 974                 item.append(delta)
 975             try:
 976                 item.append(round(d_stdev))
 977             except ValueError:
 978                 item.append(d_stdev)
 979         if rca_data:
 980             rca_nr = rca_data.get(item[0], u"-")
 981             item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-")
 982         if (len(item) == len(header)) and (item[-4] != u"NT"):
 983             tbl_lst.append(item)
 984
 985     tbl_lst = _tpc_sort_table(tbl_lst)
 986
 987     # Generate csv tables:
 988     csv_file = f"{table[u'output-file']}.csv"
 989     with open(csv_file, u"wt") as file_handler:
 990         file_handler.write(header_str)
 991         for test in tbl_lst:
 992             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 993
 994     txt_file_name = f"{table[u'output-file']}.txt"
 995     convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
 996
 997     footnote = u""
 998     with open(txt_file_name, u'a') as txt_file:
 999         txt_file.write(legend)
1000         if rca_data:
1001             footnote = rca_data.get(u"footnote", u"")
1002             if footnote:
1003                 txt_file.write(footnote)
1004         txt_file.write(u":END")
1005
1006     # Generate html table:
1007     _tpc_generate_html_table(
1008         header,
1009         tbl_lst,
1010         table[u'output-file'],
1011         legend=legend,
1012         footnote=footnote
1013     )
1014
1015
1016 def table_perf_comparison_nic(table, input_data):
1017     """Generate the table(s) with algorithm: table_perf_comparison
1018     specified in the specification file.
1019
1020     :param table: Table to generate.
1021     :param input_data: Data to process.
1022     :type table: pandas.Series
1023     :type input_data: InputData
1024     """
1025
1026     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1027
1028     # Transform the data
1029     logging.info(
1030         f"    Creating the data set for the {table.get(u'type', u'')} "
1031         f"{table.get(u'title', u'')}."
1032     )
1033     data = input_data.filter_data(table, continue_on_error=True)
1034
1035     # Prepare the header of the tables
1036     try:
1037         header = [u"Test Case", ]
1038         legend = u"\nLegend:\n"
1039
1040         rca_data = None
1041         rca = table.get(u"rca", None)
1042         if rca:
1043             try:
1044                 with open(rca.get(u"data-file", ""), u"r") as rca_file:
1045                     rca_data = load(rca_file, Loader=FullLoader)
1046                 header.insert(0, rca.get(u"title", "RCA"))
1047                 legend += (
1048                     u"RCA: Reference to the Root Cause Analysis, see below.\n"
1049                 )
1050             except (YAMLError, IOError) as err:
1051                 logging.warning(repr(err))
1052
1053         history = table.get(u"history", list())
1054         for item in history:
1055             header.extend(
1056                 [
1057                     f"{item[u'title']} Avg({table[u'include-tests']})",
1058                     f"{item[u'title']} Stdev({table[u'include-tests']})"
1059                 ]
1060             )
1061             legend += (
1062                 f"{item[u'title']} Avg({table[u'include-tests']}): "
1063                 f"Mean value of {table[u'include-tests']} [Mpps] computed from "
1064                 f"a series of runs of the listed tests executed against "
1065                 f"{item[u'title']}.\n"
1066                 f"{item[u'title']} Stdev({table[u'include-tests']}): "
1067                 f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1068                 f"computed from a series of runs of the listed tests executed "
1069                 f"against {item[u'title']}.\n"
1070             )
1071         header.extend(
1072             [
1073                 f"{table[u'reference'][u'title']} "
1074                 f"Avg({table[u'include-tests']})",
1075                 f"{table[u'reference'][u'title']} "
1076                 f"Stdev({table[u'include-tests']})",
1077                 f"{table[u'compare'][u'title']} "
1078                 f"Avg({table[u'include-tests']})",
1079                 f"{table[u'compare'][u'title']} "
1080                 f"Stdev({table[u'include-tests']})",
1081                 f"Diff({table[u'reference'][u'title']},"
1082                 f"{table[u'compare'][u'title']})",
1083                 u"Stdev(Diff)"
1084             ]
1085         )
1086         header_str = u";".join(header) + u"\n"
1087         legend += (
1088             f"{table[u'reference'][u'title']} "
1089             f"Avg({table[u'include-tests']}): "
1090             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
1091             f"series of runs of the listed tests executed against "
1092             f"{table[u'reference'][u'title']}.\n"
1093             f"{table[u'reference'][u'title']} "
1094             f"Stdev({table[u'include-tests']}): "
1095             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1096             f"computed from a series of runs of the listed tests executed "
1097             f"against {table[u'reference'][u'title']}.\n"
1098             f"{table[u'compare'][u'title']} "
1099             f"Avg({table[u'include-tests']}): "
1100             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
1101             f"series of runs of the listed tests executed against "
1102             f"{table[u'compare'][u'title']}.\n"
1103             f"{table[u'compare'][u'title']} "
1104             f"Stdev({table[u'include-tests']}): "
1105             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1106             f"computed from a series of runs of the listed tests executed "
1107             f"against {table[u'compare'][u'title']}.\n"
1108             f"Diff({table[u'reference'][u'title']},"
1109             f"{table[u'compare'][u'title']}): "
1110             f"Percentage change calculated for mean values.\n"
1111             u"Stdev(Diff): "
1112             u"Standard deviation of percentage change calculated for mean "
1113             u"values.\n"
1114             u"NT: Not Tested\n"
1115         )
1116     except (AttributeError, KeyError) as err:
1117         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1118         return
1119
1120     # Prepare data to the table:
1121     tbl_dict = dict()
1122     for job, builds in table[u"reference"][u"data"].items():
1123         for build in builds:
1124             for tst_name, tst_data in data[job][str(build)].items():
1125                 if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
1126                     continue
1127                 tst_name_mod = _tpc_modify_test_name(tst_name, ignore_nic=True)
1128                 if (u"across topologies" in table[u"title"].lower() or
1129                         (u" 3n-" in table[u"title"].lower() and
1130                          u" 2n-" in table[u"title"].lower())):
1131                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1132                 if tbl_dict.get(tst_name_mod, None) is None:
1133                     name = tst_data[u'name'].rsplit(u'-', 1)[0]
1134                     if u"across testbeds" in table[u"title"].lower() or \
1135                             u"across topologies" in table[u"title"].lower():
1136                         name = _tpc_modify_displayed_test_name(name)
1137                     tbl_dict[tst_name_mod] = {
1138                         u"name": name,
1139                         u"replace-ref": True,
1140                         u"replace-cmp": True,
1141                         u"ref-data": list(),
1142                         u"cmp-data": list()
1143                     }
1144                 _tpc_insert_data(
1145                     target=tbl_dict[tst_name_mod][u"ref-data"],
1146                     src=tst_data,
1147                     include_tests=table[u"include-tests"]
1148                 )
1149
1150     replacement = table[u"reference"].get(u"data-replacement", None)
1151     if replacement:
1152         rpl_data = input_data.filter_data(
1153             table, data=replacement, continue_on_error=True)
1154         for job, builds in replacement.items():
1155             for build in builds:
1156                 for tst_name, tst_data in rpl_data[job][str(build)].items():
1157                     if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
1158                         continue
1159                     tst_name_mod = \
1160                         _tpc_modify_test_name(tst_name, ignore_nic=True)
1161                     if (u"across topologies" in table[u"title"].lower() or
1162                             (u" 3n-" in table[u"title"].lower() and
1163                              u" 2n-" in table[u"title"].lower())):
1164                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1165                     if tbl_dict.get(tst_name_mod, None) is None:
1166                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
1167                         if u"across testbeds" in table[u"title"].lower() or \
1168                                 u"across topologies" in table[u"title"].lower():
1169                             name = _tpc_modify_displayed_test_name(name)
1170                         tbl_dict[tst_name_mod] = {
1171                             u"name": name,
1172                             u"replace-ref": False,
1173                             u"replace-cmp": True,
1174                             u"ref-data": list(),
1175                             u"cmp-data": list()
1176                         }
1177                     if tbl_dict[tst_name_mod][u"replace-ref"]:
1178                         tbl_dict[tst_name_mod][u"replace-ref"] = False
1179                         tbl_dict[tst_name_mod][u"ref-data"] = list()
1180
1181                     _tpc_insert_data(
1182                         target=tbl_dict[tst_name_mod][u"ref-data"],
1183                         src=tst_data,
1184                         include_tests=table[u"include-tests"]
1185                     )
1186
1187     for job, builds in table[u"compare"][u"data"].items():
1188         for build in builds:
1189             for tst_name, tst_data in data[job][str(build)].items():
1190                 if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
1191                     continue
1192                 tst_name_mod = _tpc_modify_test_name(tst_name, ignore_nic=True)
1193                 if (u"across topologies" in table[u"title"].lower() or
1194                         (u" 3n-" in table[u"title"].lower() and
1195                          u" 2n-" in table[u"title"].lower())):
1196                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1197                 if tbl_dict.get(tst_name_mod, None) is None:
1198                     name = tst_data[u'name'].rsplit(u'-', 1)[0]
1199                     if u"across testbeds" in table[u"title"].lower() or \
1200                             u"across topologies" in table[u"title"].lower():
1201                         name = _tpc_modify_displayed_test_name(name)
1202                     tbl_dict[tst_name_mod] = {
1203                         u"name": name,
1204                         u"replace-ref": False,
1205                         u"replace-cmp": True,
1206                         u"ref-data": list(),
1207                         u"cmp-data": list()
1208                     }
1209                 _tpc_insert_data(
1210                     target=tbl_dict[tst_name_mod][u"cmp-data"],
1211                     src=tst_data,
1212                     include_tests=table[u"include-tests"]
1213                 )
1214
1215     replacement = table[u"compare"].get(u"data-replacement", None)
1216     if replacement:
1217         rpl_data = input_data.filter_data(
1218             table, data=replacement, continue_on_error=True)
1219         for job, builds in replacement.items():
1220             for build in builds:
1221                 for tst_name, tst_data in rpl_data[job][str(build)].items():
1222                     if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
1223                         continue
1224                     tst_name_mod = \
1225                         _tpc_modify_test_name(tst_name, ignore_nic=True)
1226                     if (u"across topologies" in table[u"title"].lower() or
1227                             (u" 3n-" in table[u"title"].lower() and
1228                              u" 2n-" in table[u"title"].lower())):
1229                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1230                     if tbl_dict.get(tst_name_mod, None) is None:
1231                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
1232                         if u"across testbeds" in table[u"title"].lower() or \
1233                                 u"across topologies" in table[u"title"].lower():
1234                             name = _tpc_modify_displayed_test_name(name)
1235                         tbl_dict[tst_name_mod] = {
1236                             u"name": name,
1237                             u"replace-ref": False,
1238                             u"replace-cmp": False,
1239                             u"ref-data": list(),
1240                             u"cmp-data": list()
1241                         }
1242                     if tbl_dict[tst_name_mod][u"replace-cmp"]:
1243                         tbl_dict[tst_name_mod][u"replace-cmp"] = False
1244                         tbl_dict[tst_name_mod][u"cmp-data"] = list()
1245
1246                     _tpc_insert_data(
1247                         target=tbl_dict[tst_name_mod][u"cmp-data"],
1248                         src=tst_data,
1249                         include_tests=table[u"include-tests"]
1250                     )
1251
1252     for item in history:
1253         for job, builds in item[u"data"].items():
1254             for build in builds:
1255                 for tst_name, tst_data in data[job][str(build)].items():
1256                     if item[u"nic"] not in tst_data[u"tags"]:
1257                         continue
1258                     tst_name_mod = \
1259                         _tpc_modify_test_name(tst_name, ignore_nic=True)
1260                     if (u"across topologies" in table[u"title"].lower() or
1261                             (u" 3n-" in table[u"title"].lower() and
1262                              u" 2n-" in table[u"title"].lower())):
1263                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
1264                     if tbl_dict.get(tst_name_mod, None) is None:
1265                         continue
1266                     if tbl_dict[tst_name_mod].get(u"history", None) is None:
1267                         tbl_dict[tst_name_mod][u"history"] = OrderedDict()
1268                     if tbl_dict[tst_name_mod][u"history"].\
1269                             get(item[u"title"], None) is None:
1270                         tbl_dict[tst_name_mod][u"history"][item[
1271                             u"title"]] = list()
1272                     try:
1273                         if table[u"include-tests"] == u"MRR":
1274                             res = (tst_data[u"result"][u"receive-rate"],
1275                                    tst_data[u"result"][u"receive-stdev"])
1276                         elif table[u"include-tests"] == u"PDR":
1277                             res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
1278                         elif table[u"include-tests"] == u"NDR":
1279                             res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
1280                         else:
1281                             continue
1282                         tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
1283                             append(res)
1284                     except (TypeError, KeyError):
1285                         pass
1286
1287     tbl_lst = list()
1288     for tst_name in tbl_dict:
1289         item = [tbl_dict[tst_name][u"name"], ]
1290         if history:
1291             if tbl_dict[tst_name].get(u"history", None) is not None:
1292                 for hist_data in tbl_dict[tst_name][u"history"].values():
1293                     if hist_data:
1294                         if table[u"include-tests"] == u"MRR":
1295                             item.append(round(hist_data[0][0] / 1e6, 1))
1296                             item.append(round(hist_data[0][1] / 1e6, 1))
1297                         else:
1298                             item.append(round(mean(hist_data) / 1e6, 1))
1299                             item.append(round(stdev(hist_data) / 1e6, 1))
1300                     else:
1301                         item.extend([u"NT", u"NT"])
1302             else:
1303                 item.extend([u"NT", u"NT"])
1304         data_r = tbl_dict[tst_name][u"ref-data"]
1305         if data_r:
1306             if table[u"include-tests"] == u"MRR":
1307                 data_r_mean = data_r[0][0]
1308                 data_r_stdev = data_r[0][1]
1309             else:
1310                 data_r_mean = mean(data_r)
1311                 data_r_stdev = stdev(data_r)
1312             item.append(round(data_r_mean / 1e6, 1))
1313             item.append(round(data_r_stdev / 1e6, 1))
1314         else:
1315             data_r_mean = None
1316             data_r_stdev = None
1317             item.extend([u"NT", u"NT"])
1318         data_c = tbl_dict[tst_name][u"cmp-data"]
1319         if data_c:
1320             if table[u"include-tests"] == u"MRR":
1321                 data_c_mean = data_c[0][0]
1322                 data_c_stdev = data_c[0][1]
1323             else:
1324                 data_c_mean = mean(data_c)
1325                 data_c_stdev = stdev(data_c)
1326             item.append(round(data_c_mean / 1e6, 1))
1327             item.append(round(data_c_stdev / 1e6, 1))
1328         else:
1329             data_c_mean = None
1330             data_c_stdev = None
1331             item.extend([u"NT", u"NT"])
1332         if item[-2] == u"NT":
1333             pass
1334         elif item[-4] == u"NT":
1335             item.append(u"New in CSIT-2001")
1336             item.append(u"New in CSIT-2001")
1337         elif data_r_mean is not None and data_c_mean is not None:
1338             delta, d_stdev = relative_change_stdev(
1339                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
1340             )
1341             try:
1342                 item.append(round(delta))
1343             except ValueError:
1344                 item.append(delta)
1345             try:
1346                 item.append(round(d_stdev))
1347             except ValueError:
1348                 item.append(d_stdev)
1349         if rca_data:
1350             rca_nr = rca_data.get(item[0], u"-")
1351             item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1352         if (len(item) == len(header)) and (item[-4] != u"NT"):
1353             tbl_lst.append(item)
1354
1355     tbl_lst = _tpc_sort_table(tbl_lst)
1356
1357     # Generate csv tables:
1358     csv_file = f"{table[u'output-file']}.csv"
1359     with open(csv_file, u"wt") as file_handler:
1360         file_handler.write(header_str)
1361         for test in tbl_lst:
1362             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1363
1364     txt_file_name = f"{table[u'output-file']}.txt"
1365     convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
1366
1367     footnote = u""
1368     with open(txt_file_name, u'a') as txt_file:
1369         txt_file.write(legend)
1370         if rca_data:
1371             footnote = rca_data.get(u"footnote", u"")
1372             if footnote:
1373                 txt_file.write(footnote)
1374         txt_file.write(u":END")
1375
1376     # Generate html table:
1377     _tpc_generate_html_table(
1378         header,
1379         tbl_lst,
1380         table[u'output-file'],
1381         legend=legend,
1382         footnote=footnote
1383     )
1384
1385
1386 def table_nics_comparison(table, input_data):
1387     """Generate the table(s) with algorithm: table_nics_comparison
1388     specified in the specification file.
1389
1390     :param table: Table to generate.
1391     :param input_data: Data to process.
1392     :type table: pandas.Series
1393     :type input_data: InputData
1394     """
1395
1396     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1397
1398     # Transform the data
1399     logging.info(
1400         f"    Creating the data set for the {table.get(u'type', u'')} "
1401         f"{table.get(u'title', u'')}."
1402     )
1403     data = input_data.filter_data(table, continue_on_error=True)
1404
1405     # Prepare the header of the tables
1406     try:
1407         header = [
1408             u"Test Case",
1409             f"{table[u'reference'][u'title']} "
1410             f"Avg({table[u'include-tests']})",
1411             f"{table[u'reference'][u'title']} "
1412             f"Stdev({table[u'include-tests']})",
1413             f"{table[u'compare'][u'title']} "
1414             f"Avg({table[u'include-tests']})",
1415             f"{table[u'compare'][u'title']} "
1416             f"Stdev({table[u'include-tests']})",
1417             f"Diff({table[u'reference'][u'title']},"
1418             f"{table[u'compare'][u'title']})",
1419             u"Stdev(Diff)"
1420         ]
1421         legend = (
1422             u"\nLegend:\n"
1423             f"{table[u'reference'][u'title']} "
1424             f"Avg({table[u'include-tests']}): "
1425             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
1426             f"series of runs of the listed tests executed using "
1427             f"{table[u'reference'][u'title']} NIC.\n"
1428             f"{table[u'reference'][u'title']} "
1429             f"Stdev({table[u'include-tests']}): "
1430             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1431             f"computed from a series of runs of the listed tests executed "
1432             f"using {table[u'reference'][u'title']} NIC.\n"
1433             f"{table[u'compare'][u'title']} "
1434             f"Avg({table[u'include-tests']}): "
1435             f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
1436             f"series of runs of the listed tests executed using "
1437             f"{table[u'compare'][u'title']} NIC.\n"
1438             f"{table[u'compare'][u'title']} "
1439             f"Stdev({table[u'include-tests']}): "
1440             f"Standard deviation value of {table[u'include-tests']} [Mpps] "
1441             f"computed from a series of runs of the listed tests executed "
1442             f"using {table[u'compare'][u'title']} NIC.\n"
1443             f"Diff({table[u'reference'][u'title']},"
1444             f"{table[u'compare'][u'title']}): "
1445             f"Percentage change calculated for mean values.\n"
1446             u"Stdev(Diff): "
1447             u"Standard deviation of percentage change calculated for mean "
1448             u"values.\n"
1449             u":END"
1450         )
1451
1452     except (AttributeError, KeyError) as err:
1453         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1454         return
1455
1456     # Prepare data to the table:
1457     tbl_dict = dict()
1458     for job, builds in table[u"data"].items():
1459         for build in builds:
1460             for tst_name, tst_data in data[job][str(build)].items():
1461                 tst_name_mod = _tpc_modify_test_name(tst_name, ignore_nic=True)
1462                 if tbl_dict.get(tst_name_mod, None) is None:
1463                     name = tst_data[u'name'].rsplit(u'-', 1)[0]
1464                     tbl_dict[tst_name_mod] = {
1465                         u"name": name,
1466                         u"ref-data": list(),
1467                         u"cmp-data": list()
1468                     }
1469                 try:
1470                     if table[u"include-tests"] == u"MRR":
1471                         result = (tst_data[u"result"][u"receive-rate"],
1472                                   tst_data[u"result"][u"receive-stdev"])
1473                     elif table[u"include-tests"] == u"PDR":
1474                         result = tst_data[u"throughput"][u"PDR"][u"LOWER"]
1475                     elif table[u"include-tests"] == u"NDR":
1476                         result = tst_data[u"throughput"][u"NDR"][u"LOWER"]
1477                     else:
1478                         continue
1479
1480                     if result and \
1481                             table[u"reference"][u"nic"] in tst_data[u"tags"]:
1482                         tbl_dict[tst_name_mod][u"ref-data"].append(result)
1483                     elif result and \
1484                             table[u"compare"][u"nic"] in tst_data[u"tags"]:
1485                         tbl_dict[tst_name_mod][u"cmp-data"].append(result)
1486                 except (TypeError, KeyError) as err:
1487                     logging.debug(f"No data for {tst_name}\n{repr(err)}")
1488                     # No data in output.xml for this test
1489
1490     tbl_lst = list()
1491     for tst_name in tbl_dict:
1492         item = [tbl_dict[tst_name][u"name"], ]
1493         data_r = tbl_dict[tst_name][u"ref-data"]
1494         if data_r:
1495             if table[u"include-tests"] == u"MRR":
1496                 data_r_mean = data_r[0][0]
1497                 data_r_stdev = data_r[0][1]
1498             else:
1499                 data_r_mean = mean(data_r)
1500                 data_r_stdev = stdev(data_r)
1501             item.append(round(data_r_mean / 1e6, 1))
1502             item.append(round(data_r_stdev / 1e6, 1))
1503         else:
1504             data_r_mean = None
1505             data_r_stdev = None
1506             item.extend([None, None])
1507         data_c = tbl_dict[tst_name][u"cmp-data"]
1508         if data_c:
1509             if table[u"include-tests"] == u"MRR":
1510                 data_c_mean = data_c[0][0]
1511                 data_c_stdev = data_c[0][1]
1512             else:
1513                 data_c_mean = mean(data_c)
1514                 data_c_stdev = stdev(data_c)
1515             item.append(round(data_c_mean / 1e6, 1))
1516             item.append(round(data_c_stdev / 1e6, 1))
1517         else:
1518             data_c_mean = None
1519             data_c_stdev = None
1520             item.extend([None, None])
1521         if data_r_mean is not None and data_c_mean is not None:
1522             delta, d_stdev = relative_change_stdev(
1523                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
1524             )
1525             try:
1526                 item.append(round(delta))
1527             except ValueError:
1528                 item.append(delta)
1529             try:
1530                 item.append(round(d_stdev))
1531             except ValueError:
1532                 item.append(d_stdev)
1533             tbl_lst.append(item)
1534
1535     # Sort the table according to the relative change
1536     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1537
1538     # Generate csv tables:
1539     with open(f"{table[u'output-file']}.csv", u"wt") as file_handler:
1540         file_handler.write(u";".join(header) + u"\n")
1541         for test in tbl_lst:
1542             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1543
1544     convert_csv_to_pretty_txt(f"{table[u'output-file']}.csv",
1545                               f"{table[u'output-file']}.txt",
1546                               delimiter=u";")
1547
1548     with open(table[u'output-file'], u'a') as txt_file:
1549         txt_file.write(legend)
1550
1551     # Generate html table:
1552     _tpc_generate_html_table(
1553         header,
1554         tbl_lst,
1555         table[u'output-file'],
1556         legend=legend
1557     )
1558
1559
1560 def table_soak_vs_ndr(table, input_data):
1561     """Generate the table(s) with algorithm: table_soak_vs_ndr
1562     specified in the specification file.
1563
1564     :param table: Table to generate.
1565     :param input_data: Data to process.
1566     :type table: pandas.Series
1567     :type input_data: InputData
1568     """
1569
1570     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1571
1572     # Transform the data
1573     logging.info(
1574         f"    Creating the data set for the {table.get(u'type', u'')} "
1575         f"{table.get(u'title', u'')}."
1576     )
1577     data = input_data.filter_data(table, continue_on_error=True)
1578
1579     # Prepare the header of the table
1580     try:
1581         header = [
1582             u"Test Case",
1583             f"Avg({table[u'reference'][u'title']})",
1584             f"Stdev({table[u'reference'][u'title']})",
1585             f"Avg({table[u'compare'][u'title']})",
1586             f"Stdev{table[u'compare'][u'title']})",
1587             u"Diff",
1588             u"Stdev(Diff)"
1589         ]
1590         header_str = u";".join(header) + u"\n"
1591         legend = (
1592             u"\nLegend:\n"
1593             f"Avg({table[u'reference'][u'title']}): "
1594             f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
1595             f"from a series of runs of the listed tests.\n"
1596             f"Stdev({table[u'reference'][u'title']}): "
1597             f"Standard deviation value of {table[u'reference'][u'title']} "
1598             f"[Mpps] computed from a series of runs of the listed tests.\n"
1599             f"Avg({table[u'compare'][u'title']}): "
1600             f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
1601             f"a series of runs of the listed tests.\n"
1602             f"Stdev({table[u'compare'][u'title']}): "
1603             f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
1604             f"computed from a series of runs of the listed tests.\n"
1605             f"Diff({table[u'reference'][u'title']},"
1606             f"{table[u'compare'][u'title']}): "
1607             f"Percentage change calculated for mean values.\n"
1608             u"Stdev(Diff): "
1609             u"Standard deviation of percentage change calculated for mean "
1610             u"values.\n"
1611             u":END"
1612         )
1613     except (AttributeError, KeyError) as err:
1614         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
1615         return
1616
1617     # Create a list of available SOAK test results:
1618     tbl_dict = dict()
1619     for job, builds in table[u"compare"][u"data"].items():
1620         for build in builds:
1621             for tst_name, tst_data in data[job][str(build)].items():
1622                 if tst_data[u"type"] == u"SOAK":
1623                     tst_name_mod = tst_name.replace(u"-soak", u"")
1624                     if tbl_dict.get(tst_name_mod, None) is None:
1625                         groups = re.search(REGEX_NIC, tst_data[u"parent"])
1626                         nic = groups.group(0) if groups else u""
1627                         name = (
1628                             f"{nic}-"
1629                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
1630                         )
1631                         tbl_dict[tst_name_mod] = {
1632                             u"name": name,
1633                             u"ref-data": list(),
1634                             u"cmp-data": list()
1635                         }
1636                     try:
1637                         tbl_dict[tst_name_mod][u"cmp-data"].append(
1638                             tst_data[u"throughput"][u"LOWER"])
1639                     except (KeyError, TypeError):
1640                         pass
1641     tests_lst = tbl_dict.keys()
1642
1643     # Add corresponding NDR test results:
1644     for job, builds in table[u"reference"][u"data"].items():
1645         for build in builds:
1646             for tst_name, tst_data in data[job][str(build)].items():
1647                 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
1648                     replace(u"-mrr", u"")
1649                 if tst_name_mod not in tests_lst:
1650                     continue
1651                 try:
1652                     if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
1653                         continue
1654                     if table[u"include-tests"] == u"MRR":
1655                         result = (tst_data[u"result"][u"receive-rate"],
1656                                   tst_data[u"result"][u"receive-stdev"])
1657                     elif table[u"include-tests"] == u"PDR":
1658                         result = \
1659                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
1660                     elif table[u"include-tests"] == u"NDR":
1661                         result = \
1662                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
1663                     else:
1664                         result = None
1665                     if result is not None:
1666                         tbl_dict[tst_name_mod][u"ref-data"].append(
1667                             result)
1668                 except (KeyError, TypeError):
1669                     continue
1670
1671     tbl_lst = list()
1672     for tst_name in tbl_dict:
1673         item = [tbl_dict[tst_name][u"name"], ]
1674         data_r = tbl_dict[tst_name][u"ref-data"]
1675         if data_r:
1676             if table[u"include-tests"] == u"MRR":
1677                 data_r_mean = data_r[0][0]
1678                 data_r_stdev = data_r[0][1]
1679             else:
1680                 data_r_mean = mean(data_r)
1681                 data_r_stdev = stdev(data_r)
1682             item.append(round(data_r_mean / 1e6, 1))
1683             item.append(round(data_r_stdev / 1e6, 1))
1684         else:
1685             data_r_mean = None
1686             data_r_stdev = None
1687             item.extend([None, None])
1688         data_c = tbl_dict[tst_name][u"cmp-data"]
1689         if data_c:
1690             if table[u"include-tests"] == u"MRR":
1691                 data_c_mean = data_c[0][0]
1692                 data_c_stdev = data_c[0][1]
1693             else:
1694                 data_c_mean = mean(data_c)
1695                 data_c_stdev = stdev(data_c)
1696             item.append(round(data_c_mean / 1e6, 1))
1697             item.append(round(data_c_stdev / 1e6, 1))
1698         else:
1699             data_c_mean = None
1700             data_c_stdev = None
1701             item.extend([None, None])
1702         if data_r_mean is not None and data_c_mean is not None:
1703             delta, d_stdev = relative_change_stdev(
1704                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
1705             try:
1706                 item.append(round(delta))
1707             except ValueError:
1708                 item.append(delta)
1709             try:
1710                 item.append(round(d_stdev))
1711             except ValueError:
1712                 item.append(d_stdev)
1713             tbl_lst.append(item)
1714
1715     # Sort the table according to the relative change
1716     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1717
1718     # Generate csv tables:
1719     csv_file = f"{table[u'output-file']}.csv"
1720     with open(csv_file, u"wt") as file_handler:
1721         file_handler.write(header_str)
1722         for test in tbl_lst:
1723             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1724
1725     convert_csv_to_pretty_txt(
1726         csv_file, f"{table[u'output-file']}.txt", delimiter=u";"
1727     )
1728     with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
1729         txt_file.write(legend)
1730
1731     # Generate html table:
1732     _tpc_generate_html_table(
1733         header,
1734         tbl_lst,
1735         table[u'output-file'],
1736         legend=legend
1737     )
1738
1739
1740 def table_perf_trending_dash(table, input_data):
1741     """Generate the table(s) with algorithm:
1742     table_perf_trending_dash
1743     specified in the specification file.
1744
1745     :param table: Table to generate.
1746     :param input_data: Data to process.
1747     :type table: pandas.Series
1748     :type input_data: InputData
1749     """
1750
1751     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1752
1753     # Transform the data
1754     logging.info(
1755         f"    Creating the data set for the {table.get(u'type', u'')} "
1756         f"{table.get(u'title', u'')}."
1757     )
1758     data = input_data.filter_data(table, continue_on_error=True)
1759
1760     # Prepare the header of the tables
1761     header = [
1762         u"Test Case",
1763         u"Trend [Mpps]",
1764         u"Short-Term Change [%]",
1765         u"Long-Term Change [%]",
1766         u"Regressions [#]",
1767         u"Progressions [#]"
1768     ]
1769     header_str = u",".join(header) + u"\n"
1770
1771     # Prepare data to the table:
1772     tbl_dict = dict()
1773     for job, builds in table[u"data"].items():
1774         for build in builds:
1775             for tst_name, tst_data in data[job][str(build)].items():
1776                 if tst_name.lower() in table.get(u"ignore-list", list()):
1777                     continue
1778                 if tbl_dict.get(tst_name, None) is None:
1779                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1780                     if not groups:
1781                         continue
1782                     nic = groups.group(0)
1783                     tbl_dict[tst_name] = {
1784                         u"name": f"{nic}-{tst_data[u'name']}",
1785                         u"data": OrderedDict()
1786                     }
1787                 try:
1788                     tbl_dict[tst_name][u"data"][str(build)] = \
1789                         tst_data[u"result"][u"receive-rate"]
1790                 except (TypeError, KeyError):
1791                     pass  # No data in output.xml for this test
1792
1793     tbl_lst = list()
1794     for tst_name in tbl_dict:
1795         data_t = tbl_dict[tst_name][u"data"]
1796         if len(data_t) < 2:
1797             continue
1798
1799         classification_lst, avgs = classify_anomalies(data_t)
1800
1801         win_size = min(len(data_t), table[u"window"])
1802         long_win_size = min(len(data_t), table[u"long-trend-window"])
1803
1804         try:
1805             max_long_avg = max(
1806                 [x for x in avgs[-long_win_size:-win_size]
1807                  if not isnan(x)])
1808         except ValueError:
1809             max_long_avg = nan
1810         last_avg = avgs[-1]
1811         avg_week_ago = avgs[max(-win_size, -len(avgs))]
1812
1813         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
1814             rel_change_last = nan
1815         else:
1816             rel_change_last = round(
1817                 ((last_avg - avg_week_ago) / avg_week_ago) * 1e2, 2)
1818
1819         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
1820             rel_change_long = nan
1821         else:
1822             rel_change_long = round(
1823                 ((last_avg - max_long_avg) / max_long_avg) * 1e2, 2)
1824
1825         if classification_lst:
1826             if isnan(rel_change_last) and isnan(rel_change_long):
1827                 continue
1828             if isnan(last_avg) or isnan(rel_change_last) or \
1829                     isnan(rel_change_long):
1830                 continue
1831             tbl_lst.append(
1832                 [tbl_dict[tst_name][u"name"],
1833                  round(last_avg / 1e6, 2),
1834                  rel_change_last,
1835                  rel_change_long,
1836                  classification_lst[-win_size:].count(u"regression"),
1837                  classification_lst[-win_size:].count(u"progression")])
1838
1839     tbl_lst.sort(key=lambda rel: rel[0])
1840
1841     tbl_sorted = list()
1842     for nrr in range(table[u"window"], -1, -1):
1843         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
1844         for nrp in range(table[u"window"], -1, -1):
1845             tbl_out = [item for item in tbl_reg if item[5] == nrp]
1846             tbl_out.sort(key=lambda rel: rel[2])
1847             tbl_sorted.extend(tbl_out)
1848
1849     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1850
1851     logging.info(f"    Writing file: {file_name}")
1852     with open(file_name, u"wt") as file_handler:
1853         file_handler.write(header_str)
1854         for test in tbl_sorted:
1855             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1856
1857     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1858     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1859
1860
1861 def _generate_url(testbed, test_name):
1862     """Generate URL to a trending plot from the name of the test case.
1863
1864     :param testbed: The testbed used for testing.
1865     :param test_name: The name of the test case.
1866     :type testbed: str
1867     :type test_name: str
1868     :returns: The URL to the plot with the trending data for the given test
1869         case.
1870     :rtype str
1871     """
1872
1873     if u"x520" in test_name:
1874         nic = u"x520"
1875     elif u"x710" in test_name:
1876         nic = u"x710"
1877     elif u"xl710" in test_name:
1878         nic = u"xl710"
1879     elif u"xxv710" in test_name:
1880         nic = u"xxv710"
1881     elif u"vic1227" in test_name:
1882         nic = u"vic1227"
1883     elif u"vic1385" in test_name:
1884         nic = u"vic1385"
1885     elif u"x553" in test_name:
1886         nic = u"x553"
1887     else:
1888         nic = u""
1889
1890     if u"64b" in test_name:
1891         frame_size = u"64b"
1892     elif u"78b" in test_name:
1893         frame_size = u"78b"
1894     elif u"imix" in test_name:
1895         frame_size = u"imix"
1896     elif u"9000b" in test_name:
1897         frame_size = u"9000b"
1898     elif u"1518b" in test_name:
1899         frame_size = u"1518b"
1900     elif u"114b" in test_name:
1901         frame_size = u"114b"
1902     else:
1903         frame_size = u""
1904
1905     if u"1t1c" in test_name or \
1906         (u"-1c-" in test_name and
1907          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1908         cores = u"1t1c"
1909     elif u"2t2c" in test_name or \
1910          (u"-2c-" in test_name and
1911           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1912         cores = u"2t2c"
1913     elif u"4t4c" in test_name or \
1914          (u"-4c-" in test_name and
1915           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1916         cores = u"4t4c"
1917     elif u"2t1c" in test_name or \
1918          (u"-1c-" in test_name and
1919           testbed in (u"2n-skx", u"3n-skx")):
1920         cores = u"2t1c"
1921     elif u"4t2c" in test_name:
1922         cores = u"4t2c"
1923     elif u"8t4c" in test_name:
1924         cores = u"8t4c"
1925     else:
1926         cores = u""
1927
1928     if u"testpmd" in test_name:
1929         driver = u"testpmd"
1930     elif u"l3fwd" in test_name:
1931         driver = u"l3fwd"
1932     elif u"avf" in test_name:
1933         driver = u"avf"
1934     elif u"dnv" in testbed or u"tsh" in testbed:
1935         driver = u"ixgbe"
1936     else:
1937         driver = u"dpdk"
1938
1939     if u"acl" in test_name or \
1940             u"macip" in test_name or \
1941             u"nat" in test_name or \
1942             u"policer" in test_name or \
1943             u"cop" in test_name:
1944         bsf = u"features"
1945     elif u"scale" in test_name:
1946         bsf = u"scale"
1947     elif u"base" in test_name:
1948         bsf = u"base"
1949     else:
1950         bsf = u"base"
1951
1952     if u"114b" in test_name and u"vhost" in test_name:
1953         domain = u"vts"
1954     elif u"testpmd" in test_name or u"l3fwd" in test_name:
1955         domain = u"dpdk"
1956     elif u"memif" in test_name:
1957         domain = u"container_memif"
1958     elif u"srv6" in test_name:
1959         domain = u"srv6"
1960     elif u"vhost" in test_name:
1961         domain = u"vhost"
1962         if u"vppl2xc" in test_name:
1963             driver += u"-vpp"
1964         else:
1965             driver += u"-testpmd"
1966         if u"lbvpplacp" in test_name:
1967             bsf += u"-link-bonding"
1968     elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1969         domain = u"nf_service_density_vnfc"
1970     elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1971         domain = u"nf_service_density_cnfc"
1972     elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1973         domain = u"nf_service_density_cnfp"
1974     elif u"ipsec" in test_name:
1975         domain = u"ipsec"
1976         if u"sw" in test_name:
1977             bsf += u"-sw"
1978         elif u"hw" in test_name:
1979             bsf += u"-hw"
1980     elif u"ethip4vxlan" in test_name:
1981         domain = u"ip4_tunnels"
1982     elif u"ip4base" in test_name or u"ip4scale" in test_name:
1983         domain = u"ip4"
1984     elif u"ip6base" in test_name or u"ip6scale" in test_name:
1985         domain = u"ip6"
1986     elif u"l2xcbase" in test_name or \
1987             u"l2xcscale" in test_name or \
1988             u"l2bdbasemaclrn" in test_name or \
1989             u"l2bdscale" in test_name or \
1990             u"l2patch" in test_name:
1991         domain = u"l2"
1992     else:
1993         domain = u""
1994
1995     file_name = u"-".join((domain, testbed, nic)) + u".html#"
1996     anchor_name = u"-".join((frame_size, cores, bsf, driver))
1997
1998     return file_name + anchor_name
1999
2000
2001 def table_perf_trending_dash_html(table, input_data):
2002     """Generate the table(s) with algorithm:
2003     table_perf_trending_dash_html specified in the specification
2004     file.
2005
2006     :param table: Table to generate.
2007     :param input_data: Data to process.
2008     :type table: dict
2009     :type input_data: InputData
2010     """
2011
2012     _ = input_data
2013
2014     if not table.get(u"testbed", None):
2015         logging.error(
2016             f"The testbed is not defined for the table "
2017             f"{table.get(u'title', u'')}."
2018         )
2019         return
2020
2021     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
2022
2023     try:
2024         with open(table[u"input-file"], u'rt') as csv_file:
2025             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
2026     except KeyError:
2027         logging.warning(u"The input file is not defined.")
2028         return
2029     except csv.Error as err:
2030         logging.warning(
2031             f"Not possible to process the file {table[u'input-file']}.\n"
2032             f"{repr(err)}"
2033         )
2034         return
2035
2036     # Table:
2037     dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
2038
2039     # Table header:
2040     trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
2041     for idx, item in enumerate(csv_lst[0]):
2042         alignment = u"left" if idx == 0 else u"center"
2043         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
2044         thead.text = item
2045
2046     # Rows:
2047     colors = {
2048         u"regression": (
2049             u"#ffcccc",
2050             u"#ff9999"
2051         ),
2052         u"progression": (
2053             u"#c6ecc6",
2054             u"#9fdf9f"
2055         ),
2056         u"normal": (
2057             u"#e9f1fb",
2058             u"#d4e4f7"
2059         )
2060     }
2061     for r_idx, row in enumerate(csv_lst[1:]):
2062         if int(row[4]):
2063             color = u"regression"
2064         elif int(row[5]):
2065             color = u"progression"
2066         else:
2067             color = u"normal"
2068         trow = ET.SubElement(
2069             dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
2070         )
2071
2072         # Columns:
2073         for c_idx, item in enumerate(row):
2074             tdata = ET.SubElement(
2075                 trow,
2076                 u"td",
2077                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
2078             )
2079             # Name:
2080             if c_idx == 0:
2081                 ref = ET.SubElement(
2082                     tdata,
2083                     u"a",
2084                     attrib=dict(
2085                         href=f"../trending/"
2086                              f"{_generate_url(table.get(u'testbed', ''), item)}"
2087                     )
2088                 )
2089                 ref.text = item
2090             else:
2091                 tdata.text = item
2092     try:
2093         with open(table[u"output-file"], u'w') as html_file:
2094             logging.info(f"    Writing file: {table[u'output-file']}")
2095             html_file.write(u".. raw:: html\n\n\t")
2096             html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
2097             html_file.write(u"\n\t<p><br><br></p>\n")
2098     except KeyError:
2099         logging.warning(u"The output file is not defined.")
2100         return
2101
2102
2103 def table_last_failed_tests(table, input_data):
2104     """Generate the table(s) with algorithm: table_last_failed_tests
2105     specified in the specification file.
2106
2107     :param table: Table to generate.
2108     :param input_data: Data to process.
2109     :type table: pandas.Series
2110     :type input_data: InputData
2111     """
2112
2113     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
2114
2115     # Transform the data
2116     logging.info(
2117         f"    Creating the data set for the {table.get(u'type', u'')} "
2118         f"{table.get(u'title', u'')}."
2119     )
2120
2121     data = input_data.filter_data(table, continue_on_error=True)
2122
2123     if data is None or data.empty:
2124         logging.warning(
2125             f"    No data for the {table.get(u'type', u'')} "
2126             f"{table.get(u'title', u'')}."
2127         )
2128         return
2129
2130     tbl_list = list()
2131     for job, builds in table[u"data"].items():
2132         for build in builds:
2133             build = str(build)
2134             try:
2135                 version = input_data.metadata(job, build).get(u"version", u"")
2136             except KeyError:
2137                 logging.error(f"Data for {job}: {build} is not present.")
2138                 return
2139             tbl_list.append(build)
2140             tbl_list.append(version)
2141             failed_tests = list()
2142             passed = 0
2143             failed = 0
2144             for tst_data in data[job][build].values:
2145                 if tst_data[u"status"] != u"FAIL":
2146                     passed += 1
2147                     continue
2148                 failed += 1
2149                 groups = re.search(REGEX_NIC, tst_data[u"parent"])
2150                 if not groups:
2151                     continue
2152                 nic = groups.group(0)
2153                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
2154             tbl_list.append(str(passed))
2155             tbl_list.append(str(failed))
2156             tbl_list.extend(failed_tests)
2157
2158     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
2159     logging.info(f"    Writing file: {file_name}")
2160     with open(file_name, u"wt") as file_handler:
2161         for test in tbl_list:
2162             file_handler.write(test + u'\n')
2163
2164
2165 def table_failed_tests(table, input_data):
2166     """Generate the table(s) with algorithm: table_failed_tests
2167     specified in the specification file.
2168
2169     :param table: Table to generate.
2170     :param input_data: Data to process.
2171     :type table: pandas.Series
2172     :type input_data: InputData
2173     """
2174
2175     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
2176
2177     # Transform the data
2178     logging.info(
2179         f"    Creating the data set for the {table.get(u'type', u'')} "
2180         f"{table.get(u'title', u'')}."
2181     )
2182     data = input_data.filter_data(table, continue_on_error=True)
2183
2184     # Prepare the header of the tables
2185     header = [
2186         u"Test Case",
2187         u"Failures [#]",
2188         u"Last Failure [Time]",
2189         u"Last Failure [VPP-Build-Id]",
2190         u"Last Failure [CSIT-Job-Build-Id]"
2191     ]
2192
2193     # Generate the data for the table according to the model in the table
2194     # specification
2195
2196     now = dt.utcnow()
2197     timeperiod = timedelta(int(table.get(u"window", 7)))
2198
2199     tbl_dict = dict()
2200     for job, builds in table[u"data"].items():
2201         for build in builds:
2202             build = str(build)
2203             for tst_name, tst_data in data[job][build].items():
2204                 if tst_name.lower() in table.get(u"ignore-list", list()):
2205                     continue
2206                 if tbl_dict.get(tst_name, None) is None:
2207                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
2208                     if not groups:
2209                         continue
2210                     nic = groups.group(0)
2211                     tbl_dict[tst_name] = {
2212                         u"name": f"{nic}-{tst_data[u'name']}",
2213                         u"data": OrderedDict()
2214                     }
2215                 try:
2216                     generated = input_data.metadata(job, build).\
2217                         get(u"generated", u"")
2218                     if not generated:
2219                         continue
2220                     then = dt.strptime(generated, u"%Y%m%d %H:%M")
2221                     if (now - then) <= timeperiod:
2222                         tbl_dict[tst_name][u"data"][build] = (
2223                             tst_data[u"status"],
2224                             generated,
2225                             input_data.metadata(job, build).get(u"version",
2226                                                                 u""),
2227                             build
2228                         )
2229                 except (TypeError, KeyError) as err:
2230                     logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
2231
2232     max_fails = 0
2233     tbl_lst = list()
2234     for tst_data in tbl_dict.values():
2235         fails_nr = 0
2236         fails_last_date = u""
2237         fails_last_vpp = u""
2238         fails_last_csit = u""
2239         for val in tst_data[u"data"].values():
2240             if val[0] == u"FAIL":
2241                 fails_nr += 1
2242                 fails_last_date = val[1]
2243                 fails_last_vpp = val[2]
2244                 fails_last_csit = val[3]
2245         if fails_nr:
2246             max_fails = fails_nr if fails_nr > max_fails else max_fails
2247             tbl_lst.append(
2248                 [
2249                     tst_data[u"name"],
2250                     fails_nr,
2251                     fails_last_date,
2252                     fails_last_vpp,
2253                     f"mrr-daily-build-{fails_last_csit}"
2254                 ]
2255             )
2256
2257     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
2258     tbl_sorted = list()
2259     for nrf in range(max_fails, -1, -1):
2260         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
2261         tbl_sorted.extend(tbl_fails)
2262
2263     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
2264     logging.info(f"    Writing file: {file_name}")
2265     with open(file_name, u"wt") as file_handler:
2266         file_handler.write(u",".join(header) + u"\n")
2267         for test in tbl_sorted:
2268             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
2269
2270     logging.info(f"    Writing file: {table[u'output-file']}.txt")
2271     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
2272
2273
2274 def table_failed_tests_html(table, input_data):
2275     """Generate the table(s) with algorithm: table_failed_tests_html
2276     specified in the specification file.
2277
2278     :param table: Table to generate.
2279     :param input_data: Data to process.
2280     :type table: pandas.Series
2281     :type input_data: InputData
2282     """
2283
2284     _ = input_data
2285
2286     if not table.get(u"testbed", None):
2287         logging.error(
2288             f"The testbed is not defined for the table "
2289             f"{table.get(u'title', u'')}."
2290         )
2291         return
2292
2293     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
2294
2295     try:
2296         with open(table[u"input-file"], u'rt') as csv_file:
2297             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
2298     except KeyError:
2299         logging.warning(u"The input file is not defined.")
2300         return
2301     except csv.Error as err:
2302         logging.warning(
2303             f"Not possible to process the file {table[u'input-file']}.\n"
2304             f"{repr(err)}"
2305         )
2306         return
2307
2308     # Table:
2309     failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
2310
2311     # Table header:
2312     trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
2313     for idx, item in enumerate(csv_lst[0]):
2314         alignment = u"left" if idx == 0 else u"center"
2315         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
2316         thead.text = item
2317
2318     # Rows:
2319     colors = (u"#e9f1fb", u"#d4e4f7")
2320     for r_idx, row in enumerate(csv_lst[1:]):
2321         background = colors[r_idx % 2]
2322         trow = ET.SubElement(
2323             failed_tests, u"tr", attrib=dict(bgcolor=background)
2324         )
2325
2326         # Columns:
2327         for c_idx, item in enumerate(row):
2328             tdata = ET.SubElement(
2329                 trow,
2330                 u"td",
2331                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
2332             )
2333             # Name:
2334             if c_idx == 0:
2335                 ref = ET.SubElement(
2336                     tdata,
2337                     u"a",
2338                     attrib=dict(
2339                         href=f"../trending/"
2340                              f"{_generate_url(table.get(u'testbed', ''), item)}"
2341                     )
2342                 )
2343                 ref.text = item
2344             else:
2345                 tdata.text = item
2346     try:
2347         with open(table[u"output-file"], u'w') as html_file:
2348             logging.info(f"    Writing file: {table[u'output-file']}")
2349             html_file.write(u".. raw:: html\n\n\t")
2350             html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
2351             html_file.write(u"\n\t<p><br><br></p>\n")
2352     except KeyError:
2353         logging.warning(u"The output file is not defined.")
2354         return
2355
2356
2357 def table_comparison(table, input_data):
2358     """Generate the table(s) with algorithm: table_comparison
2359     specified in the specification file.
2360
2361     :param table: Table to generate.
2362     :param input_data: Data to process.
2363     :type table: pandas.Series
2364     :type input_data: InputData
2365     """
2366     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
2367
2368     # Transform the data
2369     logging.info(
2370         f"    Creating the data set for the {table.get(u'type', u'')} "
2371         f"{table.get(u'title', u'')}."
2372     )
2373
2374     columns = table.get(u"columns", None)
2375     if not columns:
2376         logging.error(
2377             f"No columns specified for {table.get(u'title', u'')}. Skipping."
2378         )
2379         return
2380
2381     cols = list()
2382     for idx, col in enumerate(columns):
2383         if col.get(u"data-set", None) is None:
2384             logging.warning(f"No data for column {col.get(u'title', u'')}")
2385             continue
2386         data = input_data.filter_data(
2387             table,
2388             params=[u"throughput", u"result", u"name", u"parent", u"tags"],
2389             data=col[u"data-set"],
2390             continue_on_error=True
2391         )
2392         col_data = {
2393             u"title": col.get(u"title", f"Column{idx}"),
2394             u"data": dict()
2395         }
2396         for builds in data.values:
2397             for build in builds:
2398                 for tst_name, tst_data in build.items():
2399                     tst_name_mod = \
2400                         _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
2401                     if col_data[u"data"].get(tst_name_mod, None) is None:
2402                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
2403                         if u"across testbeds" in table[u"title"].lower() or \
2404                                 u"across topologies" in table[u"title"].lower():
2405                             name = _tpc_modify_displayed_test_name(name)
2406                         col_data[u"data"][tst_name_mod] = {
2407                             u"name": name,
2408                             u"replace": True,
2409                             u"data": list(),
2410                             u"mean": None,
2411                             u"stdev": None
2412                         }
2413                     _tpc_insert_data(
2414                         target=col_data[u"data"][tst_name_mod][u"data"],
2415                         src=tst_data,
2416                         include_tests=table[u"include-tests"]
2417                     )
2418
2419         replacement = col.get(u"data-replacement", None)
2420         if replacement:
2421             rpl_data = input_data.filter_data(
2422                 table,
2423                 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
2424                 data=replacement,
2425                 continue_on_error=True
2426             )
2427             for builds in rpl_data.values:
2428                 for build in builds:
2429                     for tst_name, tst_data in build.items():
2430                         tst_name_mod = \
2431                             _tpc_modify_test_name(tst_name).\
2432                             replace(u"2n1l-", u"")
2433                         if col_data[u"data"].get(tst_name_mod, None) is None:
2434                             name = tst_data[u'name'].rsplit(u'-', 1)[0]
2435                             if u"across testbeds" in table[u"title"].lower() \
2436                                     or u"across topologies" in \
2437                                     table[u"title"].lower():
2438                                 name = _tpc_modify_displayed_test_name(name)
2439                             col_data[u"data"][tst_name_mod] = {
2440                                 u"name": name,
2441                                 u"replace": False,
2442                                 u"data": list(),
2443                                 u"mean": None,
2444                                 u"stdev": None
2445                             }
2446                         if col_data[u"data"][tst_name_mod][u"replace"]:
2447                             col_data[u"data"][tst_name_mod][u"replace"] = False
2448                             col_data[u"data"][tst_name_mod][u"data"] = list()
2449                         _tpc_insert_data(
2450                             target=col_data[u"data"][tst_name_mod][u"data"],
2451                             src=tst_data,
2452                             include_tests=table[u"include-tests"]
2453                         )
2454
2455         if table[u"include-tests"] in (u"NDR", u"PDR"):
2456             for tst_name, tst_data in col_data[u"data"].items():
2457                 if tst_data[u"data"]:
2458                     tst_data[u"mean"] = mean(tst_data[u"data"])
2459                     tst_data[u"stdev"] = stdev(tst_data[u"data"])
2460         elif table[u"include-tests"] in (u"MRR", ):
2461             for tst_name, tst_data in col_data[u"data"].items():
2462                 if tst_data[u"data"]:
2463                     tst_data[u"mean"] = tst_data[u"data"][0]
2464                     tst_data[u"stdev"] = tst_data[u"data"][0]
2465
2466         cols.append(col_data)
2467
2468     tbl_dict = dict()
2469     for col in cols:
2470         for tst_name, tst_data in col[u"data"].items():
2471             if tbl_dict.get(tst_name, None) is None:
2472                 tbl_dict[tst_name] = {
2473                     "name": tst_data[u"name"]
2474                 }
2475             tbl_dict[tst_name][col[u"title"]] = {
2476                 u"mean": tst_data[u"mean"],
2477                 u"stdev": tst_data[u"stdev"]
2478             }
2479
2480     tbl_lst = list()
2481     for tst_data in tbl_dict.values():
2482         row = [tst_data[u"name"], ]
2483         for col in cols:
2484             row.append(tst_data.get(col[u"title"], None))
2485         tbl_lst.append(row)
2486
2487     comparisons = table.get(u"comparisons", None)
2488     if comparisons and isinstance(comparisons, list):
2489         for idx, comp in enumerate(comparisons):
2490             try:
2491                 col_ref = int(comp[u"reference"])
2492                 col_cmp = int(comp[u"compare"])
2493             except KeyError:
2494                 logging.warning(u"Comparison: No references defined! Skipping.")
2495                 comparisons.pop(idx)
2496                 continue
2497             if not (0 < col_ref <= len(cols) and
2498                     0 < col_cmp <= len(cols)) or \
2499                     col_ref == col_cmp:
2500                 logging.warning(f"Wrong values of reference={col_ref} "
2501                                 f"and/or compare={col_cmp}. Skipping.")
2502                 comparisons.pop(idx)
2503                 continue
2504
2505     tbl_cmp_lst = list()
2506     if comparisons:
2507         for row in tbl_lst:
2508             new_row = deepcopy(row)
2509             add_to_tbl = False
2510             for comp in comparisons:
2511                 ref_itm = row[int(comp[u"reference"])]
2512                 if ref_itm is None and \
2513                         comp.get(u"reference-alt", None) is not None:
2514                     ref_itm = row[int(comp[u"reference-alt"])]
2515                 cmp_itm = row[int(comp[u"compare"])]
2516                 if ref_itm is not None and cmp_itm is not None and \
2517                         ref_itm[u"mean"] is not None and \
2518                         cmp_itm[u"mean"] is not None and \
2519                         ref_itm[u"stdev"] is not None and \
2520                         cmp_itm[u"stdev"] is not None:
2521                     delta, d_stdev = relative_change_stdev(
2522                         ref_itm[u"mean"], cmp_itm[u"mean"],
2523                         ref_itm[u"stdev"], cmp_itm[u"stdev"]
2524                     )
2525                     new_row.append(
2526                         {
2527                             u"mean": delta * 1e6,
2528                             u"stdev": d_stdev * 1e6
2529                         }
2530                     )
2531                     add_to_tbl = True
2532                 else:
2533                     new_row.append(None)
2534             if add_to_tbl:
2535                 tbl_cmp_lst.append(new_row)
2536
2537     tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
2538     tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
2539
2540     rcas = list()
2541     rca_in = table.get(u"rca", None)
2542     if rca_in and isinstance(rca_in, list):
2543         for idx, itm in enumerate(rca_in):
2544             try:
2545                 with open(itm.get(u"data", u""), u"r") as rca_file:
2546                     rcas.append(
2547                         {
2548                             u"title": itm.get(u"title", f"RCA{idx}"),
2549                             u"data": load(rca_file, Loader=FullLoader)
2550                         }
2551                     )
2552             except (YAMLError, IOError) as err:
2553                 logging.warning(
2554                     f"The RCA file {itm.get(u'data', u'')} does not exist or "
2555                     f"it is corrupted!"
2556                 )
2557                 logging.debug(repr(err))
2558
2559     tbl_for_csv = list()
2560     for line in tbl_cmp_lst:
2561         row = [line[0], ]
2562         for idx, itm in enumerate(line[1:]):
2563             if itm is None:
2564                 row.append(u"NT")
2565                 row.append(u"NT")
2566             else:
2567                 row.append(round(float(itm[u'mean']) / 1e6, 3))
2568                 row.append(round(float(itm[u'stdev']) / 1e6, 3))
2569         for rca in rcas:
2570             rca_nr = rca[u"data"].get(row[0], u"-")
2571             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
2572         tbl_for_csv.append(row)
2573
2574     header_csv = [u"Test Case", ]
2575     for col in cols:
2576         header_csv.append(f"Avg({col[u'title']})")
2577         header_csv.append(f"Stdev({col[u'title']})")
2578     for comp in comparisons:
2579         header_csv.append(
2580             f"Avg({comp.get(u'title', u'')}"
2581         )
2582         header_csv.append(
2583             f"Stdev({comp.get(u'title', u'')})"
2584         )
2585     header_csv.extend([rca[u"title"] for rca in rcas])
2586
2587     legend_lst = table.get(u"legend", None)
2588     if legend_lst is None:
2589         legend = u""
2590     else:
2591         legend = u"\n" + u"\n".join(legend_lst) + u"\n"
2592
2593     footnote = u""
2594     for rca in rcas:
2595         footnote += f"\n{rca[u'title']}:\n"
2596         footnote += rca[u"data"].get(u"footnote", u"")
2597
2598     csv_file = f"{table[u'output-file']}-csv.csv"
2599     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
2600         file_handler.write(
2601             u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
2602         )
2603         for test in tbl_for_csv:
2604             file_handler.write(
2605                 u",".join([f'"{item}"' for item in test]) + u"\n"
2606             )
2607         if legend_lst:
2608             for item in legend_lst:
2609                 file_handler.write(f'"{item}"\n')
2610         if footnote:
2611             for itm in footnote.split(u"\n"):
2612                 file_handler.write(f'"{itm}"\n')
2613
2614     tbl_tmp = list()
2615     max_lens = [0, ] * len(tbl_cmp_lst[0])
2616     for line in tbl_cmp_lst:
2617         row = [line[0], ]
2618         for idx, itm in enumerate(line[1:]):
2619             if itm is None:
2620                 new_itm = u"NT"
2621             else:
2622                 if idx < len(cols):
2623                     new_itm = (
2624                         f"{round(float(itm[u'mean']) / 1e6, 1)} "
2625                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
2626                         replace(u"nan", u"NaN")
2627                     )
2628                 else:
2629                     new_itm = (
2630                         f"{round(float(itm[u'mean']) / 1e6, 1):+} "
2631                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
2632                         replace(u"nan", u"NaN")
2633                     )
2634             if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]:
2635                 max_lens[idx] = len(new_itm.rsplit(u" ", 1)[-1])
2636             row.append(new_itm)
2637
2638         tbl_tmp.append(row)
2639
2640     tbl_final = list()
2641     for line in tbl_tmp:
2642         row = [line[0], ]
2643         for idx, itm in enumerate(line[1:]):
2644             if itm in (u"NT", u"NaN"):
2645                 row.append(itm)
2646                 continue
2647             itm_lst = itm.rsplit(u"\u00B1", 1)
2648             itm_lst[-1] = \
2649                 f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
2650             row.append(u"\u00B1".join(itm_lst))
2651         for rca in rcas:
2652             rca_nr = rca[u"data"].get(row[0], u"-")
2653             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
2654
2655         tbl_final.append(row)
2656
2657     header = [u"Test Case", ]
2658     header.extend([col[u"title"] for col in cols])
2659     header.extend([comp.get(u"title", u"") for comp in comparisons])
2660     header.extend([rca[u"title"] for rca in rcas])
2661
2662     # Generate csv tables:
2663     csv_file = f"{table[u'output-file']}.csv"
2664     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
2665         file_handler.write(u";".join(header) + u"\n")
2666         for test in tbl_final:
2667             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
2668
2669     # Generate txt table:
2670     txt_file_name = f"{table[u'output-file']}.txt"
2671     convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
2672
2673     with open(txt_file_name, u'a', encoding='utf-8') as txt_file:
2674         txt_file.write(legend)
2675         if footnote:
2676             txt_file.write(footnote)
2677         txt_file.write(u":END")
2678
2679     # Generate html table:
2680     _tpc_generate_html_table(
2681         header,
2682         tbl_final,
2683         table[u'output-file'],
2684         legend=legend,
2685         footnote=footnote,
2686         sort_data=False,
2687         title=table.get(u"title", u"")
2688     )