resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from collections import OrderedDict
  23 from xml.etree import ElementTree as ET
  24 from datetime import datetime as dt
  25 from datetime import timedelta
  26 from copy import deepcopy
  27
  28 import plotly.graph_objects as go
  29 import plotly.offline as ploff
  30 import pandas as pd
  31
  32 from numpy import nan, isnan
  33 from yaml import load, FullLoader, YAMLError
  34
  35 from pal_utils import mean, stdev, classify_anomalies, \
  36     convert_csv_to_pretty_txt, relative_change_stdev, relative_change
  37
  38
  39 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
  40
  41
  42 def generate_tables(spec, data):
  43     """Generate all tables specified in the specification file.
  44
  45     :param spec: Specification read from the specification file.
  46     :param data: Data to process.
  47     :type spec: Specification
  48     :type data: InputData
  49     """
  50
  51     generator = {
  52         u"table_merged_details": table_merged_details,
  53         u"table_soak_vs_ndr": table_soak_vs_ndr,
  54         u"table_perf_trending_dash": table_perf_trending_dash,
  55         u"table_perf_trending_dash_html": table_perf_trending_dash_html,
  56         u"table_last_failed_tests": table_last_failed_tests,
  57         u"table_failed_tests": table_failed_tests,
  58         u"table_failed_tests_html": table_failed_tests_html,
  59         u"table_oper_data_html": table_oper_data_html,
  60         u"table_comparison": table_comparison,
  61         u"table_weekly_comparison": table_weekly_comparison
  62     }
  63
  64     logging.info(u"Generating the tables ...")
  65     for table in spec.tables:
  66         try:
  67             if table[u"algorithm"] == u"table_weekly_comparison":
  68                 table[u"testbeds"] = spec.environment.get(u"testbeds", None)
  69             generator[table[u"algorithm"]](table, data)
  70         except NameError as err:
  71             logging.error(
  72                 f"Probably algorithm {table[u'algorithm']} is not defined: "
  73                 f"{repr(err)}"
  74             )
  75     logging.info(u"Done.")
  76
  77
  78 def table_oper_data_html(table, input_data):
  79     """Generate the table(s) with algorithm: html_table_oper_data
  80     specified in the specification file.
  81
  82     :param table: Table to generate.
  83     :param input_data: Data to process.
  84     :type table: pandas.Series
  85     :type input_data: InputData
  86     """
  87
  88     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
  89     # Transform the data
  90     logging.info(
  91         f"    Creating the data set for the {table.get(u'type', u'')} "
  92         f"{table.get(u'title', u'')}."
  93     )
  94     data = input_data.filter_data(
  95         table,
  96         params=[u"name", u"parent", u"show-run", u"type"],
  97         continue_on_error=True
  98     )
  99     if data.empty:
 100         return
 101     data = input_data.merge_data(data)
 102
 103     sort_tests = table.get(u"sort", None)
 104     if sort_tests:
 105         args = dict(
 106             inplace=True,
 107             ascending=(sort_tests == u"ascending")
 108         )
 109         data.sort_index(**args)
 110
 111     suites = input_data.filter_data(
 112         table,
 113         continue_on_error=True,
 114         data_set=u"suites"
 115     )
 116     if suites.empty:
 117         return
 118     suites = input_data.merge_data(suites)
 119
 120     def _generate_html_table(tst_data):
 121         """Generate an HTML table with operational data for the given test.
 122
 123         :param tst_data: Test data to be used to generate the table.
 124         :type tst_data: pandas.Series
 125         :returns: HTML table with operational data.
 126         :rtype: str
 127         """
 128
 129         colors = {
 130             u"header": u"#7eade7",
 131             u"empty": u"#ffffff",
 132             u"body": (u"#e9f1fb", u"#d4e4f7")
 133         }
 134
 135         tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
 136
 137         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
 138         thead = ET.SubElement(
 139             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 140         )
 141         thead.text = tst_data[u"name"]
 142
 143         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 144         thead = ET.SubElement(
 145             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 146         )
 147         thead.text = u"\t"
 148
 149         if tst_data.get(u"show-run", u"No Data") == u"No Data":
 150             trow = ET.SubElement(
 151                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 152             )
 153             tcol = ET.SubElement(
 154                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 155             )
 156             tcol.text = u"No Data"
 157
 158             trow = ET.SubElement(
 159                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 160             )
 161             thead = ET.SubElement(
 162                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 163             )
 164             font = ET.SubElement(
 165                 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 166             )
 167             font.text = u"."
 168             return str(ET.tostring(tbl, encoding=u"unicode"))
 169
 170         tbl_hdr = (
 171             u"Name",
 172             u"Nr of Vectors",
 173             u"Nr of Packets",
 174             u"Suspends",
 175             u"Cycles per Packet",
 176             u"Average Vector Size"
 177         )
 178
 179         for dut_data in tst_data[u"show-run"].values():
 180             trow = ET.SubElement(
 181                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 182             )
 183             tcol = ET.SubElement(
 184                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 185             )
 186             if dut_data.get(u"threads", None) is None:
 187                 tcol.text = u"No Data"
 188                 continue
 189
 190             bold = ET.SubElement(tcol, u"b")
 191             bold.text = (
 192                 f"Host IP: {dut_data.get(u'host', '')}, "
 193                 f"Socket: {dut_data.get(u'socket', '')}"
 194             )
 195             trow = ET.SubElement(
 196                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 197             )
 198             thead = ET.SubElement(
 199                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 200             )
 201             thead.text = u"\t"
 202
 203             for thread_nr, thread in dut_data[u"threads"].items():
 204                 trow = ET.SubElement(
 205                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 206                 )
 207                 tcol = ET.SubElement(
 208                     trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 209                 )
 210                 bold = ET.SubElement(tcol, u"b")
 211                 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
 212                 trow = ET.SubElement(
 213                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 214                 )
 215                 for idx, col in enumerate(tbl_hdr):
 216                     tcol = ET.SubElement(
 217                         trow, u"td",
 218                         attrib=dict(align=u"right" if idx else u"left")
 219                     )
 220                     font = ET.SubElement(
 221                         tcol, u"font", attrib=dict(size=u"2")
 222                     )
 223                     bold = ET.SubElement(font, u"b")
 224                     bold.text = col
 225                 for row_nr, row in enumerate(thread):
 226                     trow = ET.SubElement(
 227                         tbl, u"tr",
 228                         attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
 229                     )
 230                     for idx, col in enumerate(row):
 231                         tcol = ET.SubElement(
 232                             trow, u"td",
 233                             attrib=dict(align=u"right" if idx else u"left")
 234                         )
 235                         font = ET.SubElement(
 236                             tcol, u"font", attrib=dict(size=u"2")
 237                         )
 238                         if isinstance(col, float):
 239                             font.text = f"{col:.2f}"
 240                         else:
 241                             font.text = str(col)
 242                 trow = ET.SubElement(
 243                     tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 244                 )
 245                 thead = ET.SubElement(
 246                     trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 247                 )
 248                 thead.text = u"\t"
 249
 250         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 251         thead = ET.SubElement(
 252             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 253         )
 254         font = ET.SubElement(
 255             thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 256         )
 257         font.text = u"."
 258
 259         return str(ET.tostring(tbl, encoding=u"unicode"))
 260
 261     for suite in suites.values:
 262         html_table = str()
 263         for test_data in data.values:
 264             if test_data[u"parent"] not in suite[u"name"]:
 265                 continue
 266             html_table += _generate_html_table(test_data)
 267         if not html_table:
 268             continue
 269         try:
 270             file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
 271             with open(f"{file_name}", u'w') as html_file:
 272                 logging.info(f"    Writing file: {file_name}")
 273                 html_file.write(u".. raw:: html\n\n\t")
 274                 html_file.write(html_table)
 275                 html_file.write(u"\n\t<p><br><br></p>\n")
 276         except KeyError:
 277             logging.warning(u"The output file is not defined.")
 278             return
 279     logging.info(u"  Done.")
 280
 281
 282 def table_merged_details(table, input_data):
 283     """Generate the table(s) with algorithm: table_merged_details
 284     specified in the specification file.
 285
 286     :param table: Table to generate.
 287     :param input_data: Data to process.
 288     :type table: pandas.Series
 289     :type input_data: InputData
 290     """
 291
 292     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 293
 294     # Transform the data
 295     logging.info(
 296         f"    Creating the data set for the {table.get(u'type', u'')} "
 297         f"{table.get(u'title', u'')}."
 298     )
 299     data = input_data.filter_data(table, continue_on_error=True)
 300     data = input_data.merge_data(data)
 301
 302     sort_tests = table.get(u"sort", None)
 303     if sort_tests:
 304         args = dict(
 305             inplace=True,
 306             ascending=(sort_tests == u"ascending")
 307         )
 308         data.sort_index(**args)
 309
 310     suites = input_data.filter_data(
 311         table, continue_on_error=True, data_set=u"suites")
 312     suites = input_data.merge_data(suites)
 313
 314     # Prepare the header of the tables
 315     header = list()
 316     for column in table[u"columns"]:
 317         header.append(
 318             u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
 319         )
 320
 321     for suite in suites.values:
 322         # Generate data
 323         suite_name = suite[u"name"]
 324         table_lst = list()
 325         for test in data.keys():
 326             if data[test][u"parent"] not in suite_name:
 327                 continue
 328             row_lst = list()
 329             for column in table[u"columns"]:
 330                 try:
 331                     col_data = str(data[test][column[
 332                         u"data"].split(u" ")[1]]).replace(u'"', u'""')
 333                     # Do not include tests with "Test Failed" in test message
 334                     if u"Test Failed" in col_data:
 335                         continue
 336                     col_data = col_data.replace(
 337                         u"No Data", u"Not Captured     "
 338                     )
 339                     if column[u"data"].split(u" ")[1] in (u"name", ):
 340                         if len(col_data) > 30:
 341                             col_data_lst = col_data.split(u"-")
 342                             half = int(len(col_data_lst) / 2)
 343                             col_data = f"{u'-'.join(col_data_lst[:half])}" \
 344                                        f"- |br| " \
 345                                        f"{u'-'.join(col_data_lst[half:])}"
 346                         col_data = f" |prein| {col_data} |preout| "
 347                     elif column[u"data"].split(u" ")[1] in (u"msg", ):
 348                         # Temporary solution: remove NDR results from message:
 349                         if bool(table.get(u'remove-ndr', False)):
 350                             try:
 351                                 col_data = col_data.split(u" |br| ", 1)[1]
 352                             except IndexError:
 353                                 pass
 354                         col_data = f" |prein| {col_data} |preout| "
 355                     elif column[u"data"].split(u" ")[1] in \
 356                             (u"conf-history", u"show-run"):
 357                         col_data = col_data.replace(u" |br| ", u"", 1)
 358                         col_data = f" |prein| {col_data[:-5]} |preout| "
 359                     row_lst.append(f'"{col_data}"')
 360                 except KeyError:
 361                     row_lst.append(u'"Not captured"')
 362             if len(row_lst) == len(table[u"columns"]):
 363                 table_lst.append(row_lst)
 364
 365         # Write the data to file
 366         if table_lst:
 367             separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
 368             file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
 369             logging.info(f"      Writing file: {file_name}")
 370             with open(file_name, u"wt") as file_handler:
 371                 file_handler.write(u",".join(header) + u"\n")
 372                 for item in table_lst:
 373                     file_handler.write(u",".join(item) + u"\n")
 374
 375     logging.info(u"  Done.")
 376
 377
 378 def _tpc_modify_test_name(test_name, ignore_nic=False):
 379     """Modify a test name by replacing its parts.
 380
 381     :param test_name: Test name to be modified.
 382     :param ignore_nic: If True, NIC is removed from TC name.
 383     :type test_name: str
 384     :type ignore_nic: bool
 385     :returns: Modified test name.
 386     :rtype: str
 387     """
 388     test_name_mod = test_name.\
 389         replace(u"-ndrpdrdisc", u""). \
 390         replace(u"-ndrpdr", u"").\
 391         replace(u"-pdrdisc", u""). \
 392         replace(u"-ndrdisc", u"").\
 393         replace(u"-pdr", u""). \
 394         replace(u"-ndr", u""). \
 395         replace(u"1t1c", u"1c").\
 396         replace(u"2t1c", u"1c"). \
 397         replace(u"2t2c", u"2c").\
 398         replace(u"4t2c", u"2c"). \
 399         replace(u"4t4c", u"4c").\
 400         replace(u"8t4c", u"4c")
 401
 402     if ignore_nic:
 403         return re.sub(REGEX_NIC, u"", test_name_mod)
 404     return test_name_mod
 405
 406
 407 def _tpc_modify_displayed_test_name(test_name):
 408     """Modify a test name which is displayed in a table by replacing its parts.
 409
 410     :param test_name: Test name to be modified.
 411     :type test_name: str
 412     :returns: Modified test name.
 413     :rtype: str
 414     """
 415     return test_name.\
 416         replace(u"1t1c", u"1c").\
 417         replace(u"2t1c", u"1c"). \
 418         replace(u"2t2c", u"2c").\
 419         replace(u"4t2c", u"2c"). \
 420         replace(u"4t4c", u"4c").\
 421         replace(u"8t4c", u"4c")
 422
 423
 424 def _tpc_insert_data(target, src, include_tests):
 425     """Insert src data to the target structure.
 426
 427     :param target: Target structure where the data is placed.
 428     :param src: Source data to be placed into the target stucture.
 429     :param include_tests: Which results will be included (MRR, NDR, PDR).
 430     :type target: list
 431     :type src: dict
 432     :type include_tests: str
 433     """
 434     try:
 435         if include_tests == u"MRR":
 436             target[u"mean"] = src[u"result"][u"receive-rate"]
 437             target[u"stdev"] = src[u"result"][u"receive-stdev"]
 438         elif include_tests == u"PDR":
 439             target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
 440         elif include_tests == u"NDR":
 441             target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
 442     except (KeyError, TypeError):
 443         pass
 444
 445
 446 def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
 447                              footnote=u"", sort_data=True, title=u"",
 448                              generate_rst=True):
 449     """Generate html table from input data with simple sorting possibility.
 450
 451     :param header: Table header.
 452     :param data: Input data to be included in the table. It is a list of lists.
 453         Inner lists are rows in the table. All inner lists must be of the same
 454         length. The length of these lists must be the same as the length of the
 455         header.
 456     :param out_file_name: The name (relative or full path) where the
 457         generated html table is written.
 458     :param legend: The legend to display below the table.
 459     :param footnote: The footnote to display below the table (and legend).
 460     :param sort_data: If True the data sorting is enabled.
 461     :param title: The table (and file) title.
 462     :param generate_rst: If True, wrapping rst file is generated.
 463     :type header: list
 464     :type data: list of lists
 465     :type out_file_name: str
 466     :type legend: str
 467     :type footnote: str
 468     :type sort_data: bool
 469     :type title: str
 470     :type generate_rst: bool
 471     """
 472
 473     try:
 474         idx = header.index(u"Test Case")
 475     except ValueError:
 476         idx = 0
 477     params = {
 478         u"align-hdr": (
 479             [u"left", u"right"],
 480             [u"left", u"left", u"right"],
 481             [u"left", u"left", u"left", u"right"]
 482         ),
 483         u"align-itm": (
 484             [u"left", u"right"],
 485             [u"left", u"left", u"right"],
 486             [u"left", u"left", u"left", u"right"]
 487         ),
 488         u"width": ([15, 9], [4, 24, 10], [4, 4, 32, 10])
 489     }
 490
 491     df_data = pd.DataFrame(data, columns=header)
 492
 493     if sort_data:
 494         df_sorted = [df_data.sort_values(
 495             by=[key, header[idx]], ascending=[True, True]
 496             if key != header[idx] else [False, True]) for key in header]
 497         df_sorted_rev = [df_data.sort_values(
 498             by=[key, header[idx]], ascending=[False, True]
 499             if key != header[idx] else [True, True]) for key in header]
 500         df_sorted.extend(df_sorted_rev)
 501     else:
 502         df_sorted = df_data
 503
 504     fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
 505                    for idx in range(len(df_data))]]
 506     table_header = dict(
 507         values=[f"<b>{item.replace(u',', u',<br>')}</b>" for item in header],
 508         fill_color=u"#7eade7",
 509         align=params[u"align-hdr"][idx],
 510         font=dict(
 511             family=u"Courier New",
 512             size=12
 513         )
 514     )
 515
 516     fig = go.Figure()
 517
 518     if sort_data:
 519         for table in df_sorted:
 520             columns = [table.get(col) for col in header]
 521             fig.add_trace(
 522                 go.Table(
 523                     columnwidth=params[u"width"][idx],
 524                     header=table_header,
 525                     cells=dict(
 526                         values=columns,
 527                         fill_color=fill_color,
 528                         align=params[u"align-itm"][idx],
 529                         font=dict(
 530                             family=u"Courier New",
 531                             size=12
 532                         )
 533                     )
 534                 )
 535             )
 536
 537         buttons = list()
 538         menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
 539         menu_items.extend([f"<b>{itm}</b> (descending)" for itm in header])
 540         for idx, hdr in enumerate(menu_items):
 541             visible = [False, ] * len(menu_items)
 542             visible[idx] = True
 543             buttons.append(
 544                 dict(
 545                     label=hdr.replace(u" [Mpps]", u""),
 546                     method=u"update",
 547                     args=[{u"visible": visible}],
 548                 )
 549             )
 550
 551         fig.update_layout(
 552             updatemenus=[
 553                 go.layout.Updatemenu(
 554                     type=u"dropdown",
 555                     direction=u"down",
 556                     x=0.0,
 557                     xanchor=u"left",
 558                     y=1.002,
 559                     yanchor=u"bottom",
 560                     active=len(menu_items) - 1,
 561                     buttons=list(buttons)
 562                 )
 563             ],
 564         )
 565     else:
 566         fig.add_trace(
 567             go.Table(
 568                 columnwidth=params[u"width"][idx],
 569                 header=table_header,
 570                 cells=dict(
 571                     values=[df_sorted.get(col) for col in header],
 572                     fill_color=fill_color,
 573                     align=params[u"align-itm"][idx],
 574                     font=dict(
 575                         family=u"Courier New",
 576                         size=12
 577                     )
 578                 )
 579             )
 580         )
 581
 582     ploff.plot(
 583         fig,
 584         show_link=False,
 585         auto_open=False,
 586         filename=f"{out_file_name}_in.html"
 587     )
 588
 589     if not generate_rst:
 590         return
 591
 592     file_name = out_file_name.split(u"/")[-1]
 593     if u"vpp" in out_file_name:
 594         path = u"_tmp/src/vpp_performance_tests/comparisons/"
 595     else:
 596         path = u"_tmp/src/dpdk_performance_tests/comparisons/"
 597     with open(f"{path}{file_name}.rst", u"wt") as rst_file:
 598         rst_file.write(
 599             u"\n"
 600             u".. |br| raw:: html\n\n    <br />\n\n\n"
 601             u".. |prein| raw:: html\n\n    <pre>\n\n\n"
 602             u".. |preout| raw:: html\n\n    </pre>\n\n"
 603         )
 604         if title:
 605             rst_file.write(f"{title}\n")
 606             rst_file.write(f"{u'`' * len(title)}\n\n")
 607         rst_file.write(
 608             u".. raw:: html\n\n"
 609             f'    <iframe frameborder="0" scrolling="no" '
 610             f'width="1600" height="1200" '
 611             f'src="../..{out_file_name.replace(u"_build", u"")}_in.html">'
 612             f'</iframe>\n\n'
 613         )
 614
 615         # TODO: Use html (rst) list for legend and footnote
 616         if legend:
 617             rst_file.write(legend[1:].replace(u"\n", u" |br| "))
 618         if footnote:
 619             rst_file.write(footnote.replace(u"\n", u" |br| ")[1:])
 620
 621
 622 def table_soak_vs_ndr(table, input_data):
 623     """Generate the table(s) with algorithm: table_soak_vs_ndr
 624     specified in the specification file.
 625
 626     :param table: Table to generate.
 627     :param input_data: Data to process.
 628     :type table: pandas.Series
 629     :type input_data: InputData
 630     """
 631
 632     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 633
 634     # Transform the data
 635     logging.info(
 636         f"    Creating the data set for the {table.get(u'type', u'')} "
 637         f"{table.get(u'title', u'')}."
 638     )
 639     data = input_data.filter_data(table, continue_on_error=True)
 640
 641     # Prepare the header of the table
 642     try:
 643         header = [
 644             u"Test Case",
 645             f"Avg({table[u'reference'][u'title']})",
 646             f"Stdev({table[u'reference'][u'title']})",
 647             f"Avg({table[u'compare'][u'title']})",
 648             f"Stdev{table[u'compare'][u'title']})",
 649             u"Diff",
 650             u"Stdev(Diff)"
 651         ]
 652         header_str = u";".join(header) + u"\n"
 653         legend = (
 654             u"\nLegend:\n"
 655             f"Avg({table[u'reference'][u'title']}): "
 656             f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
 657             f"from a series of runs of the listed tests.\n"
 658             f"Stdev({table[u'reference'][u'title']}): "
 659             f"Standard deviation value of {table[u'reference'][u'title']} "
 660             f"[Mpps] computed from a series of runs of the listed tests.\n"
 661             f"Avg({table[u'compare'][u'title']}): "
 662             f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
 663             f"a series of runs of the listed tests.\n"
 664             f"Stdev({table[u'compare'][u'title']}): "
 665             f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
 666             f"computed from a series of runs of the listed tests.\n"
 667             f"Diff({table[u'reference'][u'title']},"
 668             f"{table[u'compare'][u'title']}): "
 669             f"Percentage change calculated for mean values.\n"
 670             u"Stdev(Diff): "
 671             u"Standard deviation of percentage change calculated for mean "
 672             u"values."
 673         )
 674     except (AttributeError, KeyError) as err:
 675         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 676         return
 677
 678     # Create a list of available SOAK test results:
 679     tbl_dict = dict()
 680     for job, builds in table[u"compare"][u"data"].items():
 681         for build in builds:
 682             for tst_name, tst_data in data[job][str(build)].items():
 683                 if tst_data[u"type"] == u"SOAK":
 684                     tst_name_mod = tst_name.replace(u"-soak", u"")
 685                     if tbl_dict.get(tst_name_mod, None) is None:
 686                         groups = re.search(REGEX_NIC, tst_data[u"parent"])
 687                         nic = groups.group(0) if groups else u""
 688                         name = (
 689                             f"{nic}-"
 690                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 691                         )
 692                         tbl_dict[tst_name_mod] = {
 693                             u"name": name,
 694                             u"ref-data": list(),
 695                             u"cmp-data": list()
 696                         }
 697                     try:
 698                         tbl_dict[tst_name_mod][u"cmp-data"].append(
 699                             tst_data[u"throughput"][u"LOWER"])
 700                     except (KeyError, TypeError):
 701                         pass
 702     tests_lst = tbl_dict.keys()
 703
 704     # Add corresponding NDR test results:
 705     for job, builds in table[u"reference"][u"data"].items():
 706         for build in builds:
 707             for tst_name, tst_data in data[job][str(build)].items():
 708                 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
 709                     replace(u"-mrr", u"")
 710                 if tst_name_mod not in tests_lst:
 711                     continue
 712                 try:
 713                     if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
 714                         continue
 715                     if table[u"include-tests"] == u"MRR":
 716                         result = (tst_data[u"result"][u"receive-rate"],
 717                                   tst_data[u"result"][u"receive-stdev"])
 718                     elif table[u"include-tests"] == u"PDR":
 719                         result = \
 720                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
 721                     elif table[u"include-tests"] == u"NDR":
 722                         result = \
 723                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
 724                     else:
 725                         result = None
 726                     if result is not None:
 727                         tbl_dict[tst_name_mod][u"ref-data"].append(
 728                             result)
 729                 except (KeyError, TypeError):
 730                     continue
 731
 732     tbl_lst = list()
 733     for tst_name in tbl_dict:
 734         item = [tbl_dict[tst_name][u"name"], ]
 735         data_r = tbl_dict[tst_name][u"ref-data"]
 736         if data_r:
 737             if table[u"include-tests"] == u"MRR":
 738                 data_r_mean = data_r[0][0]
 739                 data_r_stdev = data_r[0][1]
 740             else:
 741                 data_r_mean = mean(data_r)
 742                 data_r_stdev = stdev(data_r)
 743             item.append(round(data_r_mean / 1e6, 1))
 744             item.append(round(data_r_stdev / 1e6, 1))
 745         else:
 746             data_r_mean = None
 747             data_r_stdev = None
 748             item.extend([None, None])
 749         data_c = tbl_dict[tst_name][u"cmp-data"]
 750         if data_c:
 751             if table[u"include-tests"] == u"MRR":
 752                 data_c_mean = data_c[0][0]
 753                 data_c_stdev = data_c[0][1]
 754             else:
 755                 data_c_mean = mean(data_c)
 756                 data_c_stdev = stdev(data_c)
 757             item.append(round(data_c_mean / 1e6, 1))
 758             item.append(round(data_c_stdev / 1e6, 1))
 759         else:
 760             data_c_mean = None
 761             data_c_stdev = None
 762             item.extend([None, None])
 763         if data_r_mean is not None and data_c_mean is not None:
 764             delta, d_stdev = relative_change_stdev(
 765                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
 766             try:
 767                 item.append(round(delta))
 768             except ValueError:
 769                 item.append(delta)
 770             try:
 771                 item.append(round(d_stdev))
 772             except ValueError:
 773                 item.append(d_stdev)
 774             tbl_lst.append(item)
 775
 776     # Sort the table according to the relative change
 777     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 778
 779     # Generate csv tables:
 780     csv_file = f"{table[u'output-file']}.csv"
 781     with open(csv_file, u"wt") as file_handler:
 782         file_handler.write(header_str)
 783         for test in tbl_lst:
 784             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 785
 786     convert_csv_to_pretty_txt(
 787         csv_file, f"{table[u'output-file']}.txt", delimiter=u";"
 788     )
 789     with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
 790         txt_file.write(legend)
 791
 792     # Generate html table:
 793     _tpc_generate_html_table(
 794         header,
 795         tbl_lst,
 796         table[u'output-file'],
 797         legend=legend,
 798         title=table.get(u"title", u"")
 799     )
 800
 801
 802 def table_perf_trending_dash(table, input_data):
 803     """Generate the table(s) with algorithm:
 804     table_perf_trending_dash
 805     specified in the specification file.
 806
 807     :param table: Table to generate.
 808     :param input_data: Data to process.
 809     :type table: pandas.Series
 810     :type input_data: InputData
 811     """
 812
 813     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 814
 815     # Transform the data
 816     logging.info(
 817         f"    Creating the data set for the {table.get(u'type', u'')} "
 818         f"{table.get(u'title', u'')}."
 819     )
 820     data = input_data.filter_data(table, continue_on_error=True)
 821
 822     # Prepare the header of the tables
 823     header = [
 824         u"Test Case",
 825         u"Trend [Mpps]",
 826         u"Short-Term Change [%]",
 827         u"Long-Term Change [%]",
 828         u"Regressions [#]",
 829         u"Progressions [#]"
 830     ]
 831     header_str = u",".join(header) + u"\n"
 832
 833     incl_tests = table.get(u"include-tests", u"MRR")
 834
 835     # Prepare data to the table:
 836     tbl_dict = dict()
 837     for job, builds in table[u"data"].items():
 838         for build in builds:
 839             for tst_name, tst_data in data[job][str(build)].items():
 840                 if tst_name.lower() in table.get(u"ignore-list", list()):
 841                     continue
 842                 if tbl_dict.get(tst_name, None) is None:
 843                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
 844                     if not groups:
 845                         continue
 846                     nic = groups.group(0)
 847                     tbl_dict[tst_name] = {
 848                         u"name": f"{nic}-{tst_data[u'name']}",
 849                         u"data": OrderedDict()
 850                     }
 851                 try:
 852                     if incl_tests == u"MRR":
 853                         tbl_dict[tst_name][u"data"][str(build)] = \
 854                             tst_data[u"result"][u"receive-rate"]
 855                     elif incl_tests == u"NDR":
 856                         tbl_dict[tst_name][u"data"][str(build)] = \
 857                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
 858                     elif incl_tests == u"PDR":
 859                         tbl_dict[tst_name][u"data"][str(build)] = \
 860                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
 861                 except (TypeError, KeyError):
 862                     pass  # No data in output.xml for this test
 863
 864     tbl_lst = list()
 865     for tst_name in tbl_dict:
 866         data_t = tbl_dict[tst_name][u"data"]
 867         if len(data_t) < 2:
 868             continue
 869
 870         classification_lst, avgs, _ = classify_anomalies(data_t)
 871
 872         win_size = min(len(data_t), table[u"window"])
 873         long_win_size = min(len(data_t), table[u"long-trend-window"])
 874
 875         try:
 876             max_long_avg = max(
 877                 [x for x in avgs[-long_win_size:-win_size]
 878                  if not isnan(x)])
 879         except ValueError:
 880             max_long_avg = nan
 881         last_avg = avgs[-1]
 882         avg_week_ago = avgs[max(-win_size, -len(avgs))]
 883
 884         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
 885             rel_change_last = nan
 886         else:
 887             rel_change_last = round(
 888                 ((last_avg - avg_week_ago) / avg_week_ago) * 1e2, 2)
 889
 890         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
 891             rel_change_long = nan
 892         else:
 893             rel_change_long = round(
 894                 ((last_avg - max_long_avg) / max_long_avg) * 1e2, 2)
 895
 896         if classification_lst:
 897             if isnan(rel_change_last) and isnan(rel_change_long):
 898                 continue
 899             if isnan(last_avg) or isnan(rel_change_last) or \
 900                     isnan(rel_change_long):
 901                 continue
 902             tbl_lst.append(
 903                 [tbl_dict[tst_name][u"name"],
 904                  round(last_avg / 1e6, 2),
 905                  rel_change_last,
 906                  rel_change_long,
 907                  classification_lst[-win_size+1:].count(u"regression"),
 908                  classification_lst[-win_size+1:].count(u"progression")])
 909
 910     tbl_lst.sort(key=lambda rel: rel[0])
 911
 912     tbl_sorted = list()
 913     for nrr in range(table[u"window"], -1, -1):
 914         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
 915         for nrp in range(table[u"window"], -1, -1):
 916             tbl_out = [item for item in tbl_reg if item[5] == nrp]
 917             tbl_out.sort(key=lambda rel: rel[2])
 918             tbl_sorted.extend(tbl_out)
 919
 920     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
 921
 922     logging.info(f"    Writing file: {file_name}")
 923     with open(file_name, u"wt") as file_handler:
 924         file_handler.write(header_str)
 925         for test in tbl_sorted:
 926             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
 927
 928     logging.info(f"    Writing file: {table[u'output-file']}.txt")
 929     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
 930
 931
 932 def _generate_url(testbed, test_name):
 933     """Generate URL to a trending plot from the name of the test case.
 934
 935     :param testbed: The testbed used for testing.
 936     :param test_name: The name of the test case.
 937     :type testbed: str
 938     :type test_name: str
 939     :returns: The URL to the plot with the trending data for the given test
 940         case.
 941     :rtype str
 942     """
 943
 944     if u"x520" in test_name:
 945         nic = u"x520"
 946     elif u"x710" in test_name:
 947         nic = u"x710"
 948     elif u"xl710" in test_name:
 949         nic = u"xl710"
 950     elif u"xxv710" in test_name:
 951         nic = u"xxv710"
 952     elif u"vic1227" in test_name:
 953         nic = u"vic1227"
 954     elif u"vic1385" in test_name:
 955         nic = u"vic1385"
 956     elif u"x553" in test_name:
 957         nic = u"x553"
 958     elif u"cx556" in test_name or u"cx556a" in test_name:
 959         nic = u"cx556a"
 960     else:
 961         nic = u""
 962
 963     if u"64b" in test_name:
 964         frame_size = u"64b"
 965     elif u"78b" in test_name:
 966         frame_size = u"78b"
 967     elif u"imix" in test_name:
 968         frame_size = u"imix"
 969     elif u"9000b" in test_name:
 970         frame_size = u"9000b"
 971     elif u"1518b" in test_name:
 972         frame_size = u"1518b"
 973     elif u"114b" in test_name:
 974         frame_size = u"114b"
 975     else:
 976         frame_size = u""
 977
 978     if u"1t1c" in test_name or \
 979         (u"-1c-" in test_name and
 980          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 981         cores = u"1t1c"
 982     elif u"2t2c" in test_name or \
 983          (u"-2c-" in test_name and
 984           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 985         cores = u"2t2c"
 986     elif u"4t4c" in test_name or \
 987          (u"-4c-" in test_name and
 988           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 989         cores = u"4t4c"
 990     elif u"2t1c" in test_name or \
 991          (u"-1c-" in test_name and
 992           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
 993         cores = u"2t1c"
 994     elif u"4t2c" in test_name or \
 995          (u"-2c-" in test_name and
 996           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
 997         cores = u"4t2c"
 998     elif u"8t4c" in test_name or \
 999          (u"-4c-" in test_name and
1000           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1001         cores = u"8t4c"
1002     else:
1003         cores = u""
1004
1005     if u"testpmd" in test_name:
1006         driver = u"testpmd"
1007     elif u"l3fwd" in test_name:
1008         driver = u"l3fwd"
1009     elif u"avf" in test_name:
1010         driver = u"avf"
1011     elif u"rdma" in test_name:
1012         driver = u"rdma"
1013     elif u"dnv" in testbed or u"tsh" in testbed:
1014         driver = u"ixgbe"
1015     else:
1016         driver = u"dpdk"
1017
1018     if u"macip-iacl1s" in test_name:
1019         bsf = u"features-macip-iacl1"
1020     elif u"macip-iacl10s" in test_name:
1021         bsf = u"features-macip-iacl01"
1022     elif u"macip-iacl50s" in test_name:
1023         bsf = u"features-macip-iacl50"
1024     elif u"iacl1s" in test_name:
1025         bsf = u"features-iacl1"
1026     elif u"iacl10s" in test_name:
1027         bsf = u"features-iacl10"
1028     elif u"iacl50s" in test_name:
1029         bsf = u"features-iacl50"
1030     elif u"oacl1s" in test_name:
1031         bsf = u"features-oacl1"
1032     elif u"oacl10s" in test_name:
1033         bsf = u"features-oacl10"
1034     elif u"oacl50s" in test_name:
1035         bsf = u"features-oacl50"
1036     elif u"udpsrcscale" in test_name:
1037         bsf = u"features-udp"
1038     elif u"iacl" in test_name:
1039         bsf = u"features"
1040     elif u"policer" in test_name:
1041         bsf = u"features"
1042     elif u"cop" in test_name:
1043         bsf = u"features"
1044     elif u"nat" in test_name:
1045         bsf = u"features"
1046     elif u"macip" in test_name:
1047         bsf = u"features"
1048     elif u"scale" in test_name:
1049         bsf = u"scale"
1050     elif u"base" in test_name:
1051         bsf = u"base"
1052     else:
1053         bsf = u"base"
1054
1055     if u"114b" in test_name and u"vhost" in test_name:
1056         domain = u"vts"
1057     elif u"testpmd" in test_name or u"l3fwd" in test_name:
1058         domain = u"dpdk"
1059     elif u"memif" in test_name:
1060         domain = u"container_memif"
1061     elif u"srv6" in test_name:
1062         domain = u"srv6"
1063     elif u"vhost" in test_name:
1064         domain = u"vhost"
1065         if u"vppl2xc" in test_name:
1066             driver += u"-vpp"
1067         else:
1068             driver += u"-testpmd"
1069         if u"lbvpplacp" in test_name:
1070             bsf += u"-link-bonding"
1071     elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1072         domain = u"nf_service_density_vnfc"
1073     elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1074         domain = u"nf_service_density_cnfc"
1075     elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1076         domain = u"nf_service_density_cnfp"
1077     elif u"ipsec" in test_name:
1078         domain = u"ipsec"
1079         if u"sw" in test_name:
1080             bsf += u"-sw"
1081         elif u"hw" in test_name:
1082             bsf += u"-hw"
1083     elif u"ethip4vxlan" in test_name:
1084         domain = u"ip4_tunnels"
1085     elif u"ip4base" in test_name or u"ip4scale" in test_name:
1086         domain = u"ip4"
1087     elif u"ip6base" in test_name or u"ip6scale" in test_name:
1088         domain = u"ip6"
1089     elif u"l2xcbase" in test_name or \
1090             u"l2xcscale" in test_name or \
1091             u"l2bdbasemaclrn" in test_name or \
1092             u"l2bdscale" in test_name or \
1093             u"l2patch" in test_name:
1094         domain = u"l2"
1095     else:
1096         domain = u""
1097
1098     file_name = u"-".join((domain, testbed, nic)) + u".html#"
1099     anchor_name = u"-".join((frame_size, cores, bsf, driver))
1100
1101     return file_name + anchor_name
1102
1103
1104 def table_perf_trending_dash_html(table, input_data):
1105     """Generate the table(s) with algorithm:
1106     table_perf_trending_dash_html specified in the specification
1107     file.
1108
1109     :param table: Table to generate.
1110     :param input_data: Data to process.
1111     :type table: dict
1112     :type input_data: InputData
1113     """
1114
1115     _ = input_data
1116
1117     if not table.get(u"testbed", None):
1118         logging.error(
1119             f"The testbed is not defined for the table "
1120             f"{table.get(u'title', u'')}. Skipping."
1121         )
1122         return
1123
1124     test_type = table.get(u"test-type", u"MRR")
1125     if test_type not in (u"MRR", u"NDR", u"PDR"):
1126         logging.error(
1127             f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
1128             f"Skipping."
1129         )
1130         return
1131
1132     if test_type in (u"NDR", u"PDR"):
1133         lnk_dir = u"../ndrpdr_trending/"
1134         lnk_sufix = f"-{test_type.lower()}"
1135     else:
1136         lnk_dir = u"../trending/"
1137         lnk_sufix = u""
1138
1139     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1140
1141     try:
1142         with open(table[u"input-file"], u'rt') as csv_file:
1143             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1144     except KeyError:
1145         logging.warning(u"The input file is not defined.")
1146         return
1147     except csv.Error as err:
1148         logging.warning(
1149             f"Not possible to process the file {table[u'input-file']}.\n"
1150             f"{repr(err)}"
1151         )
1152         return
1153
1154     # Table:
1155     dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1156
1157     # Table header:
1158     trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1159     for idx, item in enumerate(csv_lst[0]):
1160         alignment = u"left" if idx == 0 else u"center"
1161         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1162         thead.text = item
1163
1164     # Rows:
1165     colors = {
1166         u"regression": (
1167             u"#ffcccc",
1168             u"#ff9999"
1169         ),
1170         u"progression": (
1171             u"#c6ecc6",
1172             u"#9fdf9f"
1173         ),
1174         u"normal": (
1175             u"#e9f1fb",
1176             u"#d4e4f7"
1177         )
1178     }
1179     for r_idx, row in enumerate(csv_lst[1:]):
1180         if int(row[4]):
1181             color = u"regression"
1182         elif int(row[5]):
1183             color = u"progression"
1184         else:
1185             color = u"normal"
1186         trow = ET.SubElement(
1187             dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
1188         )
1189
1190         # Columns:
1191         for c_idx, item in enumerate(row):
1192             tdata = ET.SubElement(
1193                 trow,
1194                 u"td",
1195                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1196             )
1197             # Name:
1198             if c_idx == 0 and table.get(u"add-links", True):
1199                 ref = ET.SubElement(
1200                     tdata,
1201                     u"a",
1202                     attrib=dict(
1203                         href=f"{lnk_dir}"
1204                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1205                              f"{lnk_sufix}"
1206                     )
1207                 )
1208                 ref.text = item
1209             else:
1210                 tdata.text = item
1211     try:
1212         with open(table[u"output-file"], u'w') as html_file:
1213             logging.info(f"    Writing file: {table[u'output-file']}")
1214             html_file.write(u".. raw:: html\n\n\t")
1215             html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
1216             html_file.write(u"\n\t<p><br><br></p>\n")
1217     except KeyError:
1218         logging.warning(u"The output file is not defined.")
1219         return
1220
1221
1222 def table_last_failed_tests(table, input_data):
1223     """Generate the table(s) with algorithm: table_last_failed_tests
1224     specified in the specification file.
1225
1226     :param table: Table to generate.
1227     :param input_data: Data to process.
1228     :type table: pandas.Series
1229     :type input_data: InputData
1230     """
1231
1232     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1233
1234     # Transform the data
1235     logging.info(
1236         f"    Creating the data set for the {table.get(u'type', u'')} "
1237         f"{table.get(u'title', u'')}."
1238     )
1239
1240     data = input_data.filter_data(table, continue_on_error=True)
1241
1242     if data is None or data.empty:
1243         logging.warning(
1244             f"    No data for the {table.get(u'type', u'')} "
1245             f"{table.get(u'title', u'')}."
1246         )
1247         return
1248
1249     tbl_list = list()
1250     for job, builds in table[u"data"].items():
1251         for build in builds:
1252             build = str(build)
1253             try:
1254                 version = input_data.metadata(job, build).get(u"version", u"")
1255             except KeyError:
1256                 logging.error(f"Data for {job}: {build} is not present.")
1257                 return
1258             tbl_list.append(build)
1259             tbl_list.append(version)
1260             failed_tests = list()
1261             passed = 0
1262             failed = 0
1263             for tst_data in data[job][build].values:
1264                 if tst_data[u"status"] != u"FAIL":
1265                     passed += 1
1266                     continue
1267                 failed += 1
1268                 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1269                 if not groups:
1270                     continue
1271                 nic = groups.group(0)
1272                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
1273             tbl_list.append(str(passed))
1274             tbl_list.append(str(failed))
1275             tbl_list.extend(failed_tests)
1276
1277     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1278     logging.info(f"    Writing file: {file_name}")
1279     with open(file_name, u"wt") as file_handler:
1280         for test in tbl_list:
1281             file_handler.write(test + u'\n')
1282
1283
1284 def table_failed_tests(table, input_data):
1285     """Generate the table(s) with algorithm: table_failed_tests
1286     specified in the specification file.
1287
1288     :param table: Table to generate.
1289     :param input_data: Data to process.
1290     :type table: pandas.Series
1291     :type input_data: InputData
1292     """
1293
1294     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1295
1296     # Transform the data
1297     logging.info(
1298         f"    Creating the data set for the {table.get(u'type', u'')} "
1299         f"{table.get(u'title', u'')}."
1300     )
1301     data = input_data.filter_data(table, continue_on_error=True)
1302
1303     test_type = u"MRR"
1304     if u"NDRPDR" in table.get(u"filter", list()):
1305         test_type = u"NDRPDR"
1306
1307     # Prepare the header of the tables
1308     header = [
1309         u"Test Case",
1310         u"Failures [#]",
1311         u"Last Failure [Time]",
1312         u"Last Failure [VPP-Build-Id]",
1313         u"Last Failure [CSIT-Job-Build-Id]"
1314     ]
1315
1316     # Generate the data for the table according to the model in the table
1317     # specification
1318
1319     now = dt.utcnow()
1320     timeperiod = timedelta(int(table.get(u"window", 7)))
1321
1322     tbl_dict = dict()
1323     for job, builds in table[u"data"].items():
1324         for build in builds:
1325             build = str(build)
1326             for tst_name, tst_data in data[job][build].items():
1327                 if tst_name.lower() in table.get(u"ignore-list", list()):
1328                     continue
1329                 if tbl_dict.get(tst_name, None) is None:
1330                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1331                     if not groups:
1332                         continue
1333                     nic = groups.group(0)
1334                     tbl_dict[tst_name] = {
1335                         u"name": f"{nic}-{tst_data[u'name']}",
1336                         u"data": OrderedDict()
1337                     }
1338                 try:
1339                     generated = input_data.metadata(job, build).\
1340                         get(u"generated", u"")
1341                     if not generated:
1342                         continue
1343                     then = dt.strptime(generated, u"%Y%m%d %H:%M")
1344                     if (now - then) <= timeperiod:
1345                         tbl_dict[tst_name][u"data"][build] = (
1346                             tst_data[u"status"],
1347                             generated,
1348                             input_data.metadata(job, build).get(u"version",
1349                                                                 u""),
1350                             build
1351                         )
1352                 except (TypeError, KeyError) as err:
1353                     logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
1354
1355     max_fails = 0
1356     tbl_lst = list()
1357     for tst_data in tbl_dict.values():
1358         fails_nr = 0
1359         fails_last_date = u""
1360         fails_last_vpp = u""
1361         fails_last_csit = u""
1362         for val in tst_data[u"data"].values():
1363             if val[0] == u"FAIL":
1364                 fails_nr += 1
1365                 fails_last_date = val[1]
1366                 fails_last_vpp = val[2]
1367                 fails_last_csit = val[3]
1368         if fails_nr:
1369             max_fails = fails_nr if fails_nr > max_fails else max_fails
1370             tbl_lst.append([
1371                 tst_data[u"name"],
1372                 fails_nr,
1373                 fails_last_date,
1374                 fails_last_vpp,
1375                 f"{u'mrr-daily' if test_type == u'MRR' else u'ndrpdr-weekly'}"
1376                 f"-build-{fails_last_csit}"
1377             ])
1378
1379     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1380     tbl_sorted = list()
1381     for nrf in range(max_fails, -1, -1):
1382         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1383         tbl_sorted.extend(tbl_fails)
1384
1385     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1386     logging.info(f"    Writing file: {file_name}")
1387     with open(file_name, u"wt") as file_handler:
1388         file_handler.write(u",".join(header) + u"\n")
1389         for test in tbl_sorted:
1390             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1391
1392     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1393     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1394
1395
1396 def table_failed_tests_html(table, input_data):
1397     """Generate the table(s) with algorithm: table_failed_tests_html
1398     specified in the specification file.
1399
1400     :param table: Table to generate.
1401     :param input_data: Data to process.
1402     :type table: pandas.Series
1403     :type input_data: InputData
1404     """
1405
1406     _ = input_data
1407
1408     if not table.get(u"testbed", None):
1409         logging.error(
1410             f"The testbed is not defined for the table "
1411             f"{table.get(u'title', u'')}. Skipping."
1412         )
1413         return
1414
1415     test_type = table.get(u"test-type", u"MRR")
1416     if test_type not in (u"MRR", u"NDR", u"PDR", u"NDRPDR"):
1417         logging.error(
1418             f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
1419             f"Skipping."
1420         )
1421         return
1422
1423     if test_type in (u"NDRPDR", u"NDR", u"PDR"):
1424         lnk_dir = u"../ndrpdr_trending/"
1425         lnk_sufix = u"-pdr"
1426     else:
1427         lnk_dir = u"../trending/"
1428         lnk_sufix = u""
1429
1430     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1431
1432     try:
1433         with open(table[u"input-file"], u'rt') as csv_file:
1434             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1435     except KeyError:
1436         logging.warning(u"The input file is not defined.")
1437         return
1438     except csv.Error as err:
1439         logging.warning(
1440             f"Not possible to process the file {table[u'input-file']}.\n"
1441             f"{repr(err)}"
1442         )
1443         return
1444
1445     # Table:
1446     failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1447
1448     # Table header:
1449     trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1450     for idx, item in enumerate(csv_lst[0]):
1451         alignment = u"left" if idx == 0 else u"center"
1452         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1453         thead.text = item
1454
1455     # Rows:
1456     colors = (u"#e9f1fb", u"#d4e4f7")
1457     for r_idx, row in enumerate(csv_lst[1:]):
1458         background = colors[r_idx % 2]
1459         trow = ET.SubElement(
1460             failed_tests, u"tr", attrib=dict(bgcolor=background)
1461         )
1462
1463         # Columns:
1464         for c_idx, item in enumerate(row):
1465             tdata = ET.SubElement(
1466                 trow,
1467                 u"td",
1468                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1469             )
1470             # Name:
1471             if c_idx == 0 and table.get(u"add-links", True):
1472                 ref = ET.SubElement(
1473                     tdata,
1474                     u"a",
1475                     attrib=dict(
1476                         href=f"{lnk_dir}"
1477                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1478                              f"{lnk_sufix}"
1479                     )
1480                 )
1481                 ref.text = item
1482             else:
1483                 tdata.text = item
1484     try:
1485         with open(table[u"output-file"], u'w') as html_file:
1486             logging.info(f"    Writing file: {table[u'output-file']}")
1487             html_file.write(u".. raw:: html\n\n\t")
1488             html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
1489             html_file.write(u"\n\t<p><br><br></p>\n")
1490     except KeyError:
1491         logging.warning(u"The output file is not defined.")
1492         return
1493
1494
1495 def table_comparison(table, input_data):
1496     """Generate the table(s) with algorithm: table_comparison
1497     specified in the specification file.
1498
1499     :param table: Table to generate.
1500     :param input_data: Data to process.
1501     :type table: pandas.Series
1502     :type input_data: InputData
1503     """
1504     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1505
1506     # Transform the data
1507     logging.info(
1508         f"    Creating the data set for the {table.get(u'type', u'')} "
1509         f"{table.get(u'title', u'')}."
1510     )
1511
1512     columns = table.get(u"columns", None)
1513     if not columns:
1514         logging.error(
1515             f"No columns specified for {table.get(u'title', u'')}. Skipping."
1516         )
1517         return
1518
1519     cols = list()
1520     for idx, col in enumerate(columns):
1521         if col.get(u"data-set", None) is None:
1522             logging.warning(f"No data for column {col.get(u'title', u'')}")
1523             continue
1524         tag = col.get(u"tag", None)
1525         data = input_data.filter_data(
1526             table,
1527             params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1528             data=col[u"data-set"],
1529             continue_on_error=True
1530         )
1531         col_data = {
1532             u"title": col.get(u"title", f"Column{idx}"),
1533             u"data": dict()
1534         }
1535         for builds in data.values:
1536             for build in builds:
1537                 for tst_name, tst_data in build.items():
1538                     if tag and tag not in tst_data[u"tags"]:
1539                         continue
1540                     tst_name_mod = \
1541                         _tpc_modify_test_name(tst_name, ignore_nic=True).\
1542                         replace(u"2n1l-", u"")
1543                     if col_data[u"data"].get(tst_name_mod, None) is None:
1544                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
1545                         if u"across testbeds" in table[u"title"].lower() or \
1546                                 u"across topologies" in table[u"title"].lower():
1547                             name = _tpc_modify_displayed_test_name(name)
1548                         col_data[u"data"][tst_name_mod] = {
1549                             u"name": name,
1550                             u"replace": True,
1551                             u"data": list(),
1552                             u"mean": None,
1553                             u"stdev": None
1554                         }
1555                     _tpc_insert_data(
1556                         target=col_data[u"data"][tst_name_mod],
1557                         src=tst_data,
1558                         include_tests=table[u"include-tests"]
1559                     )
1560
1561         replacement = col.get(u"data-replacement", None)
1562         if replacement:
1563             rpl_data = input_data.filter_data(
1564                 table,
1565                 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1566                 data=replacement,
1567                 continue_on_error=True
1568             )
1569             for builds in rpl_data.values:
1570                 for build in builds:
1571                     for tst_name, tst_data in build.items():
1572                         if tag and tag not in tst_data[u"tags"]:
1573                             continue
1574                         tst_name_mod = \
1575                             _tpc_modify_test_name(tst_name, ignore_nic=True).\
1576                             replace(u"2n1l-", u"")
1577                         if col_data[u"data"].get(tst_name_mod, None) is None:
1578                             name = tst_data[u'name'].rsplit(u'-', 1)[0]
1579                             if u"across testbeds" in table[u"title"].lower() \
1580                                     or u"across topologies" in \
1581                                     table[u"title"].lower():
1582                                 name = _tpc_modify_displayed_test_name(name)
1583                             col_data[u"data"][tst_name_mod] = {
1584                                 u"name": name,
1585                                 u"replace": False,
1586                                 u"data": list(),
1587                                 u"mean": None,
1588                                 u"stdev": None
1589                             }
1590                         if col_data[u"data"][tst_name_mod][u"replace"]:
1591                             col_data[u"data"][tst_name_mod][u"replace"] = False
1592                             col_data[u"data"][tst_name_mod][u"data"] = list()
1593                         _tpc_insert_data(
1594                             target=col_data[u"data"][tst_name_mod],
1595                             src=tst_data,
1596                             include_tests=table[u"include-tests"]
1597                         )
1598
1599         if table[u"include-tests"] in (u"NDR", u"PDR"):
1600             for tst_name, tst_data in col_data[u"data"].items():
1601                 if tst_data[u"data"]:
1602                     tst_data[u"mean"] = mean(tst_data[u"data"])
1603                     tst_data[u"stdev"] = stdev(tst_data[u"data"])
1604
1605         cols.append(col_data)
1606
1607     tbl_dict = dict()
1608     for col in cols:
1609         for tst_name, tst_data in col[u"data"].items():
1610             if tbl_dict.get(tst_name, None) is None:
1611                 tbl_dict[tst_name] = {
1612                     "name": tst_data[u"name"]
1613                 }
1614             tbl_dict[tst_name][col[u"title"]] = {
1615                 u"mean": tst_data[u"mean"],
1616                 u"stdev": tst_data[u"stdev"]
1617             }
1618
1619     if not tbl_dict:
1620         logging.warning(f"No data for table {table.get(u'title', u'')}!")
1621         return
1622
1623     tbl_lst = list()
1624     for tst_data in tbl_dict.values():
1625         row = [tst_data[u"name"], ]
1626         for col in cols:
1627             row.append(tst_data.get(col[u"title"], None))
1628         tbl_lst.append(row)
1629
1630     comparisons = table.get(u"comparisons", None)
1631     if comparisons and isinstance(comparisons, list):
1632         for idx, comp in enumerate(comparisons):
1633             try:
1634                 col_ref = int(comp[u"reference"])
1635                 col_cmp = int(comp[u"compare"])
1636             except KeyError:
1637                 logging.warning(u"Comparison: No references defined! Skipping.")
1638                 comparisons.pop(idx)
1639                 continue
1640             if not (0 < col_ref <= len(cols) and
1641                     0 < col_cmp <= len(cols)) or \
1642                     col_ref == col_cmp:
1643                 logging.warning(f"Wrong values of reference={col_ref} "
1644                                 f"and/or compare={col_cmp}. Skipping.")
1645                 comparisons.pop(idx)
1646                 continue
1647
1648     tbl_cmp_lst = list()
1649     if comparisons:
1650         for row in tbl_lst:
1651             new_row = deepcopy(row)
1652             add_to_tbl = False
1653             for comp in comparisons:
1654                 ref_itm = row[int(comp[u"reference"])]
1655                 if ref_itm is None and \
1656                         comp.get(u"reference-alt", None) is not None:
1657                     ref_itm = row[int(comp[u"reference-alt"])]
1658                 cmp_itm = row[int(comp[u"compare"])]
1659                 if ref_itm is not None and cmp_itm is not None and \
1660                         ref_itm[u"mean"] is not None and \
1661                         cmp_itm[u"mean"] is not None and \
1662                         ref_itm[u"stdev"] is not None and \
1663                         cmp_itm[u"stdev"] is not None:
1664                     delta, d_stdev = relative_change_stdev(
1665                         ref_itm[u"mean"], cmp_itm[u"mean"],
1666                         ref_itm[u"stdev"], cmp_itm[u"stdev"]
1667                     )
1668                     new_row.append(
1669                         {
1670                             u"mean": delta * 1e6,
1671                             u"stdev": d_stdev * 1e6
1672                         }
1673                     )
1674                     add_to_tbl = True
1675                 else:
1676                     new_row.append(None)
1677             if add_to_tbl:
1678                 tbl_cmp_lst.append(new_row)
1679
1680     tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
1681     tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
1682
1683     rcas = list()
1684     rca_in = table.get(u"rca", None)
1685     if rca_in and isinstance(rca_in, list):
1686         for idx, itm in enumerate(rca_in):
1687             try:
1688                 with open(itm.get(u"data", u""), u"r") as rca_file:
1689                     rcas.append(
1690                         {
1691                             u"title": itm.get(u"title", f"RCA{idx}"),
1692                             u"data": load(rca_file, Loader=FullLoader)
1693                         }
1694                     )
1695             except (YAMLError, IOError) as err:
1696                 logging.warning(
1697                     f"The RCA file {itm.get(u'data', u'')} does not exist or "
1698                     f"it is corrupted!"
1699                 )
1700                 logging.debug(repr(err))
1701
1702     tbl_for_csv = list()
1703     for line in tbl_cmp_lst:
1704         row = [line[0], ]
1705         for idx, itm in enumerate(line[1:]):
1706             if itm is None:
1707                 row.append(u"NT")
1708                 row.append(u"NT")
1709             else:
1710                 row.append(round(float(itm[u'mean']) / 1e6, 3))
1711                 row.append(round(float(itm[u'stdev']) / 1e6, 3))
1712         for rca in rcas:
1713             rca_nr = rca[u"data"].get(row[0], u"-")
1714             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1715         tbl_for_csv.append(row)
1716
1717     header_csv = [u"Test Case", ]
1718     for col in cols:
1719         header_csv.append(f"Avg({col[u'title']})")
1720         header_csv.append(f"Stdev({col[u'title']})")
1721     for comp in comparisons:
1722         header_csv.append(
1723             f"Avg({comp.get(u'title', u'')})"
1724         )
1725         header_csv.append(
1726             f"Stdev({comp.get(u'title', u'')})"
1727         )
1728     header_csv.extend([rca[u"title"] for rca in rcas])
1729
1730     legend_lst = table.get(u"legend", None)
1731     if legend_lst is None:
1732         legend = u""
1733     else:
1734         legend = u"\n" + u"\n".join(legend_lst) + u"\n"
1735
1736     footnote = u""
1737     for rca in rcas:
1738         footnote += f"\n{rca[u'title']}:\n"
1739         footnote += rca[u"data"].get(u"footnote", u"")
1740
1741     csv_file = f"{table[u'output-file']}-csv.csv"
1742     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1743         file_handler.write(
1744             u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
1745         )
1746         for test in tbl_for_csv:
1747             file_handler.write(
1748                 u",".join([f'"{item}"' for item in test]) + u"\n"
1749             )
1750         if legend_lst:
1751             for item in legend_lst:
1752                 file_handler.write(f'"{item}"\n')
1753         if footnote:
1754             for itm in footnote.split(u"\n"):
1755                 file_handler.write(f'"{itm}"\n')
1756
1757     tbl_tmp = list()
1758     max_lens = [0, ] * len(tbl_cmp_lst[0])
1759     for line in tbl_cmp_lst:
1760         row = [line[0], ]
1761         for idx, itm in enumerate(line[1:]):
1762             if itm is None:
1763                 new_itm = u"NT"
1764             else:
1765                 if idx < len(cols):
1766                     new_itm = (
1767                         f"{round(float(itm[u'mean']) / 1e6, 1)} "
1768                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1769                         replace(u"nan", u"NaN")
1770                     )
1771                 else:
1772                     new_itm = (
1773                         f"{round(float(itm[u'mean']) / 1e6, 1):+} "
1774                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1775                         replace(u"nan", u"NaN")
1776                     )
1777             if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]:
1778                 max_lens[idx] = len(new_itm.rsplit(u" ", 1)[-1])
1779             row.append(new_itm)
1780
1781         tbl_tmp.append(row)
1782
1783     tbl_final = list()
1784     for line in tbl_tmp:
1785         row = [line[0], ]
1786         for idx, itm in enumerate(line[1:]):
1787             if itm in (u"NT", u"NaN"):
1788                 row.append(itm)
1789                 continue
1790             itm_lst = itm.rsplit(u"\u00B1", 1)
1791             itm_lst[-1] = \
1792                 f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
1793             row.append(u"\u00B1".join(itm_lst))
1794         for rca in rcas:
1795             rca_nr = rca[u"data"].get(row[0], u"-")
1796             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1797
1798         tbl_final.append(row)
1799
1800     header = [u"Test Case", ]
1801     header.extend([col[u"title"] for col in cols])
1802     header.extend([comp.get(u"title", u"") for comp in comparisons])
1803     header.extend([rca[u"title"] for rca in rcas])
1804
1805     # Generate csv tables:
1806     csv_file = f"{table[u'output-file']}.csv"
1807     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1808         file_handler.write(u";".join(header) + u"\n")
1809         for test in tbl_final:
1810             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1811
1812     # Generate txt table:
1813     txt_file_name = f"{table[u'output-file']}.txt"
1814     convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
1815
1816     with open(txt_file_name, u'a', encoding='utf-8') as txt_file:
1817         txt_file.write(legend)
1818         txt_file.write(footnote)
1819
1820     # Generate html table:
1821     _tpc_generate_html_table(
1822         header,
1823         tbl_final,
1824         table[u'output-file'],
1825         legend=legend,
1826         footnote=footnote,
1827         sort_data=False,
1828         title=table.get(u"title", u"")
1829     )
1830
1831
1832 def table_weekly_comparison(table, in_data):
1833     """Generate the table(s) with algorithm: table_weekly_comparison
1834     specified in the specification file.
1835
1836     :param table: Table to generate.
1837     :param in_data: Data to process.
1838     :type table: pandas.Series
1839     :type in_data: InputData
1840     """
1841     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1842
1843     # Transform the data
1844     logging.info(
1845         f"    Creating the data set for the {table.get(u'type', u'')} "
1846         f"{table.get(u'title', u'')}."
1847     )
1848
1849     incl_tests = table.get(u"include-tests", None)
1850     if incl_tests not in (u"NDR", u"PDR"):
1851         logging.error(f"Wrong tests to include specified ({incl_tests}).")
1852         return
1853
1854     nr_cols = table.get(u"nr-of-data-columns", None)
1855     if not nr_cols or nr_cols < 2:
1856         logging.error(
1857             f"No columns specified for {table.get(u'title', u'')}. Skipping."
1858         )
1859         return
1860
1861     data = in_data.filter_data(
1862         table,
1863         params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1864         continue_on_error=True
1865     )
1866
1867     header = [
1868         [u"VPP Version", ],
1869         [u"Start Timestamp", ],
1870         [u"CSIT Build", ],
1871         [u"CSIT Testbed", ]
1872     ]
1873     tbl_dict = dict()
1874     idx = 0
1875     tb_tbl = table.get(u"testbeds", None)
1876     for job_name, job_data in data.items():
1877         for build_nr, build in job_data.items():
1878             if idx >= nr_cols:
1879                 break
1880             if build.empty:
1881                 continue
1882
1883             tb_ip = in_data.metadata(job_name, build_nr).get(u"testbed", u"")
1884             if tb_ip and tb_tbl:
1885                 testbed = tb_tbl.get(tb_ip, u"")
1886             else:
1887                 testbed = u""
1888             header[2].insert(1, build_nr)
1889             header[3].insert(1, testbed)
1890             header[1].insert(
1891                 1, in_data.metadata(job_name, build_nr).get(u"generated", u"")
1892             )
1893             header[0].insert(
1894                 1, in_data.metadata(job_name, build_nr).get(u"version", u"")
1895             )
1896
1897             for tst_name, tst_data in build.items():
1898                 tst_name_mod = \
1899                     _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
1900                 if not tbl_dict.get(tst_name_mod, None):
1901                     tbl_dict[tst_name_mod] = dict(
1902                         name=tst_data[u'name'].rsplit(u'-', 1)[0],
1903                     )
1904                 try:
1905                     tbl_dict[tst_name_mod][-idx - 1] = \
1906                         tst_data[u"throughput"][incl_tests][u"LOWER"]
1907                 except (TypeError, IndexError, KeyError, ValueError):
1908                     pass
1909             idx += 1
1910
1911     if idx < nr_cols:
1912         logging.error(u"Not enough data to build the table! Skipping")
1913         return
1914
1915     cmp_dict = dict()
1916     for idx, cmp in enumerate(table.get(u"comparisons", list())):
1917         idx_ref = cmp.get(u"reference", None)
1918         idx_cmp = cmp.get(u"compare", None)
1919         if idx_ref is None or idx_cmp is None:
1920             continue
1921         header[0].append(
1922             f"Diff({header[0][idx_ref - idx].split(u'~')[-1]} vs "
1923             f"{header[0][idx_cmp - idx].split(u'~')[-1]})"
1924         )
1925         header[1].append(u"")
1926         header[2].append(u"")
1927         header[3].append(u"")
1928         for tst_name, tst_data in tbl_dict.items():
1929             if not cmp_dict.get(tst_name, None):
1930                 cmp_dict[tst_name] = list()
1931             ref_data = tst_data.get(idx_ref, None)
1932             cmp_data = tst_data.get(idx_cmp, None)
1933             if ref_data is None or cmp_data is None:
1934                 cmp_dict[tst_name].append(float('nan'))
1935             else:
1936                 cmp_dict[tst_name].append(
1937                     relative_change(ref_data, cmp_data)
1938                 )
1939
1940     tbl_lst = list()
1941     for tst_name, tst_data in tbl_dict.items():
1942         itm_lst = [tst_data[u"name"], ]
1943         for idx in range(nr_cols):
1944             item = tst_data.get(-idx - 1, None)
1945             if item is None:
1946                 itm_lst.insert(1, None)
1947             else:
1948                 itm_lst.insert(1, round(item / 1e6, 1))
1949         itm_lst.extend(
1950             [
1951                 None if itm is None else round(itm, 1)
1952                 for itm in cmp_dict[tst_name]
1953             ]
1954         )
1955         tbl_lst.append(itm_lst)
1956
1957     tbl_lst.sort(key=lambda rel: rel[0], reverse=False)
1958     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1959
1960     # Generate csv table:
1961     csv_file = f"{table[u'output-file']}.csv"
1962     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1963         for hdr in header:
1964             file_handler.write(u",".join(hdr) + u"\n")
1965         for test in tbl_lst:
1966             file_handler.write(u",".join(
1967                 [
1968                     str(item).replace(u"None", u"-").replace(u"nan", u"-").
1969                     replace(u"null", u"-") for item in test
1970                 ]
1971             ) + u"\n")
1972
1973     txt_file = f"{table[u'output-file']}.txt"
1974     convert_csv_to_pretty_txt(csv_file, txt_file, delimiter=u",")
1975
1976     # Reorganize header in txt table
1977     txt_table = list()
1978     with open(txt_file, u"rt", encoding='utf-8') as file_handler:
1979         for line in file_handler:
1980             txt_table.append(line)
1981     try:
1982         txt_table.insert(5, txt_table.pop(2))
1983         with open(txt_file, u"wt", encoding='utf-8') as file_handler:
1984             file_handler.writelines(txt_table)
1985     except IndexError:
1986         pass
1987
1988     # Generate html table:
1989     hdr_html = [
1990         u"<br>".join(row) for row in zip(*header)
1991     ]
1992     _tpc_generate_html_table(
1993         hdr_html,
1994         tbl_lst,
1995         table[u'output-file'],
1996         sort_data=True,
1997         title=table.get(u"title", u""),
1998         generate_rst=False
1999     )