resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from collections import OrderedDict
  23 from xml.etree import ElementTree as ET
  24 from datetime import datetime as dt
  25 from datetime import timedelta
  26 from copy import deepcopy
  27
  28 import plotly.graph_objects as go
  29 import plotly.offline as ploff
  30 import pandas as pd
  31
  32 from numpy import nan, isnan
  33 from yaml import load, FullLoader, YAMLError
  34
  35 from pal_utils import mean, stdev, classify_anomalies, \
  36     convert_csv_to_pretty_txt, relative_change_stdev, relative_change
  37
  38
  39 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
  40
  41
  42 def generate_tables(spec, data):
  43     """Generate all tables specified in the specification file.
  44
  45     :param spec: Specification read from the specification file.
  46     :param data: Data to process.
  47     :type spec: Specification
  48     :type data: InputData
  49     """
  50
  51     generator = {
  52         u"table_merged_details": table_merged_details,
  53         u"table_soak_vs_ndr": table_soak_vs_ndr,
  54         u"table_perf_trending_dash": table_perf_trending_dash,
  55         u"table_perf_trending_dash_html": table_perf_trending_dash_html,
  56         u"table_last_failed_tests": table_last_failed_tests,
  57         u"table_failed_tests": table_failed_tests,
  58         u"table_failed_tests_html": table_failed_tests_html,
  59         u"table_oper_data_html": table_oper_data_html,
  60         u"table_comparison": table_comparison,
  61         u"table_weekly_comparison": table_weekly_comparison
  62     }
  63
  64     logging.info(u"Generating the tables ...")
  65     for table in spec.tables:
  66         try:
  67             if table[u"algorithm"] == u"table_weekly_comparison":
  68                 table[u"testbeds"] = spec.environment.get(u"testbeds", None)
  69             generator[table[u"algorithm"]](table, data)
  70         except NameError as err:
  71             logging.error(
  72                 f"Probably algorithm {table[u'algorithm']} is not defined: "
  73                 f"{repr(err)}"
  74             )
  75     logging.info(u"Done.")
  76
  77
  78 def table_oper_data_html(table, input_data):
  79     """Generate the table(s) with algorithm: html_table_oper_data
  80     specified in the specification file.
  81
  82     :param table: Table to generate.
  83     :param input_data: Data to process.
  84     :type table: pandas.Series
  85     :type input_data: InputData
  86     """
  87
  88     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
  89     # Transform the data
  90     logging.info(
  91         f"    Creating the data set for the {table.get(u'type', u'')} "
  92         f"{table.get(u'title', u'')}."
  93     )
  94     data = input_data.filter_data(
  95         table,
  96         params=[u"name", u"parent", u"show-run", u"type"],
  97         continue_on_error=True
  98     )
  99     if data.empty:
 100         return
 101     data = input_data.merge_data(data)
 102
 103     sort_tests = table.get(u"sort", None)
 104     if sort_tests:
 105         args = dict(
 106             inplace=True,
 107             ascending=(sort_tests == u"ascending")
 108         )
 109         data.sort_index(**args)
 110
 111     suites = input_data.filter_data(
 112         table,
 113         continue_on_error=True,
 114         data_set=u"suites"
 115     )
 116     if suites.empty:
 117         return
 118     suites = input_data.merge_data(suites)
 119
 120     def _generate_html_table(tst_data):
 121         """Generate an HTML table with operational data for the given test.
 122
 123         :param tst_data: Test data to be used to generate the table.
 124         :type tst_data: pandas.Series
 125         :returns: HTML table with operational data.
 126         :rtype: str
 127         """
 128
 129         colors = {
 130             u"header": u"#7eade7",
 131             u"empty": u"#ffffff",
 132             u"body": (u"#e9f1fb", u"#d4e4f7")
 133         }
 134
 135         tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
 136
 137         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
 138         thead = ET.SubElement(
 139             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 140         )
 141         thead.text = tst_data[u"name"]
 142
 143         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 144         thead = ET.SubElement(
 145             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 146         )
 147         thead.text = u"\t"
 148
 149         if tst_data.get(u"show-run", u"No Data") == u"No Data":
 150             trow = ET.SubElement(
 151                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 152             )
 153             tcol = ET.SubElement(
 154                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 155             )
 156             tcol.text = u"No Data"
 157
 158             trow = ET.SubElement(
 159                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 160             )
 161             thead = ET.SubElement(
 162                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 163             )
 164             font = ET.SubElement(
 165                 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 166             )
 167             font.text = u"."
 168             return str(ET.tostring(tbl, encoding=u"unicode"))
 169
 170         tbl_hdr = (
 171             u"Name",
 172             u"Nr of Vectors",
 173             u"Nr of Packets",
 174             u"Suspends",
 175             u"Cycles per Packet",
 176             u"Average Vector Size"
 177         )
 178
 179         for dut_data in tst_data[u"show-run"].values():
 180             trow = ET.SubElement(
 181                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 182             )
 183             tcol = ET.SubElement(
 184                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 185             )
 186             if dut_data.get(u"threads", None) is None:
 187                 tcol.text = u"No Data"
 188                 continue
 189
 190             bold = ET.SubElement(tcol, u"b")
 191             bold.text = (
 192                 f"Host IP: {dut_data.get(u'host', '')}, "
 193                 f"Socket: {dut_data.get(u'socket', '')}"
 194             )
 195             trow = ET.SubElement(
 196                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 197             )
 198             thead = ET.SubElement(
 199                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 200             )
 201             thead.text = u"\t"
 202
 203             for thread_nr, thread in dut_data[u"threads"].items():
 204                 trow = ET.SubElement(
 205                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 206                 )
 207                 tcol = ET.SubElement(
 208                     trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 209                 )
 210                 bold = ET.SubElement(tcol, u"b")
 211                 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
 212                 trow = ET.SubElement(
 213                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 214                 )
 215                 for idx, col in enumerate(tbl_hdr):
 216                     tcol = ET.SubElement(
 217                         trow, u"td",
 218                         attrib=dict(align=u"right" if idx else u"left")
 219                     )
 220                     font = ET.SubElement(
 221                         tcol, u"font", attrib=dict(size=u"2")
 222                     )
 223                     bold = ET.SubElement(font, u"b")
 224                     bold.text = col
 225                 for row_nr, row in enumerate(thread):
 226                     trow = ET.SubElement(
 227                         tbl, u"tr",
 228                         attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
 229                     )
 230                     for idx, col in enumerate(row):
 231                         tcol = ET.SubElement(
 232                             trow, u"td",
 233                             attrib=dict(align=u"right" if idx else u"left")
 234                         )
 235                         font = ET.SubElement(
 236                             tcol, u"font", attrib=dict(size=u"2")
 237                         )
 238                         if isinstance(col, float):
 239                             font.text = f"{col:.2f}"
 240                         else:
 241                             font.text = str(col)
 242                 trow = ET.SubElement(
 243                     tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 244                 )
 245                 thead = ET.SubElement(
 246                     trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 247                 )
 248                 thead.text = u"\t"
 249
 250         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 251         thead = ET.SubElement(
 252             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 253         )
 254         font = ET.SubElement(
 255             thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 256         )
 257         font.text = u"."
 258
 259         return str(ET.tostring(tbl, encoding=u"unicode"))
 260
 261     for suite in suites.values:
 262         html_table = str()
 263         for test_data in data.values:
 264             if test_data[u"parent"] not in suite[u"name"]:
 265                 continue
 266             html_table += _generate_html_table(test_data)
 267         if not html_table:
 268             continue
 269         try:
 270             file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
 271             with open(f"{file_name}", u'w') as html_file:
 272                 logging.info(f"    Writing file: {file_name}")
 273                 html_file.write(u".. raw:: html\n\n\t")
 274                 html_file.write(html_table)
 275                 html_file.write(u"\n\t<p><br><br></p>\n")
 276         except KeyError:
 277             logging.warning(u"The output file is not defined.")
 278             return
 279     logging.info(u"  Done.")
 280
 281
 282 def table_merged_details(table, input_data):
 283     """Generate the table(s) with algorithm: table_merged_details
 284     specified in the specification file.
 285
 286     :param table: Table to generate.
 287     :param input_data: Data to process.
 288     :type table: pandas.Series
 289     :type input_data: InputData
 290     """
 291
 292     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 293
 294     # Transform the data
 295     logging.info(
 296         f"    Creating the data set for the {table.get(u'type', u'')} "
 297         f"{table.get(u'title', u'')}."
 298     )
 299     data = input_data.filter_data(table, continue_on_error=True)
 300     data = input_data.merge_data(data)
 301
 302     sort_tests = table.get(u"sort", None)
 303     if sort_tests:
 304         args = dict(
 305             inplace=True,
 306             ascending=(sort_tests == u"ascending")
 307         )
 308         data.sort_index(**args)
 309
 310     suites = input_data.filter_data(
 311         table, continue_on_error=True, data_set=u"suites")
 312     suites = input_data.merge_data(suites)
 313
 314     # Prepare the header of the tables
 315     header = list()
 316     for column in table[u"columns"]:
 317         header.append(
 318             u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
 319         )
 320
 321     for suite in suites.values:
 322         # Generate data
 323         suite_name = suite[u"name"]
 324         table_lst = list()
 325         for test in data.keys():
 326             if data[test][u"parent"] not in suite_name:
 327                 continue
 328             row_lst = list()
 329             for column in table[u"columns"]:
 330                 try:
 331                     col_data = str(data[test][column[
 332                         u"data"].split(u" ")[1]]).replace(u'"', u'""')
 333                     # Do not include tests with "Test Failed" in test message
 334                     if u"Test Failed" in col_data:
 335                         continue
 336                     col_data = col_data.replace(
 337                         u"No Data", u"Not Captured     "
 338                     )
 339                     if column[u"data"].split(u" ")[1] in (u"name", ):
 340                         if len(col_data) > 30:
 341                             col_data_lst = col_data.split(u"-")
 342                             half = int(len(col_data_lst) / 2)
 343                             col_data = f"{u'-'.join(col_data_lst[:half])}" \
 344                                        f"- |br| " \
 345                                        f"{u'-'.join(col_data_lst[half:])}"
 346                         col_data = f" |prein| {col_data} |preout| "
 347                     elif column[u"data"].split(u" ")[1] in (u"msg", ):
 348                         # Temporary solution: remove NDR results from message:
 349                         if bool(table.get(u'remove-ndr', False)):
 350                             try:
 351                                 col_data = col_data.split(u" |br| ", 1)[1]
 352                             except IndexError:
 353                                 pass
 354                         col_data = f" |prein| {col_data} |preout| "
 355                     elif column[u"data"].split(u" ")[1] in \
 356                             (u"conf-history", u"show-run"):
 357                         col_data = col_data.replace(u" |br| ", u"", 1)
 358                         col_data = f" |prein| {col_data[:-5]} |preout| "
 359                     row_lst.append(f'"{col_data}"')
 360                 except KeyError:
 361                     row_lst.append(u'"Not captured"')
 362             if len(row_lst) == len(table[u"columns"]):
 363                 table_lst.append(row_lst)
 364
 365         # Write the data to file
 366         if table_lst:
 367             separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
 368             file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
 369             logging.info(f"      Writing file: {file_name}")
 370             with open(file_name, u"wt") as file_handler:
 371                 file_handler.write(u",".join(header) + u"\n")
 372                 for item in table_lst:
 373                     file_handler.write(u",".join(item) + u"\n")
 374
 375     logging.info(u"  Done.")
 376
 377
 378 def _tpc_modify_test_name(test_name, ignore_nic=False):
 379     """Modify a test name by replacing its parts.
 380
 381     :param test_name: Test name to be modified.
 382     :param ignore_nic: If True, NIC is removed from TC name.
 383     :type test_name: str
 384     :type ignore_nic: bool
 385     :returns: Modified test name.
 386     :rtype: str
 387     """
 388     test_name_mod = test_name.\
 389         replace(u"-ndrpdrdisc", u""). \
 390         replace(u"-ndrpdr", u"").\
 391         replace(u"-pdrdisc", u""). \
 392         replace(u"-ndrdisc", u"").\
 393         replace(u"-pdr", u""). \
 394         replace(u"-ndr", u""). \
 395         replace(u"1t1c", u"1c").\
 396         replace(u"2t1c", u"1c"). \
 397         replace(u"2t2c", u"2c").\
 398         replace(u"4t2c", u"2c"). \
 399         replace(u"4t4c", u"4c").\
 400         replace(u"8t4c", u"4c")
 401
 402     if ignore_nic:
 403         return re.sub(REGEX_NIC, u"", test_name_mod)
 404     return test_name_mod
 405
 406
 407 def _tpc_modify_displayed_test_name(test_name):
 408     """Modify a test name which is displayed in a table by replacing its parts.
 409
 410     :param test_name: Test name to be modified.
 411     :type test_name: str
 412     :returns: Modified test name.
 413     :rtype: str
 414     """
 415     return test_name.\
 416         replace(u"1t1c", u"1c").\
 417         replace(u"2t1c", u"1c"). \
 418         replace(u"2t2c", u"2c").\
 419         replace(u"4t2c", u"2c"). \
 420         replace(u"4t4c", u"4c").\
 421         replace(u"8t4c", u"4c")
 422
 423
 424 def _tpc_insert_data(target, src, include_tests):
 425     """Insert src data to the target structure.
 426
 427     :param target: Target structure where the data is placed.
 428     :param src: Source data to be placed into the target stucture.
 429     :param include_tests: Which results will be included (MRR, NDR, PDR).
 430     :type target: list
 431     :type src: dict
 432     :type include_tests: str
 433     """
 434     try:
 435         if include_tests == u"MRR":
 436             target[u"mean"] = src[u"result"][u"receive-rate"]
 437             target[u"stdev"] = src[u"result"][u"receive-stdev"]
 438         elif include_tests == u"PDR":
 439             target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
 440         elif include_tests == u"NDR":
 441             target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
 442     except (KeyError, TypeError):
 443         pass
 444
 445
 446 def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
 447                              footnote=u"", sort_data=True, title=u"",
 448                              generate_rst=True):
 449     """Generate html table from input data with simple sorting possibility.
 450
 451     :param header: Table header.
 452     :param data: Input data to be included in the table. It is a list of lists.
 453         Inner lists are rows in the table. All inner lists must be of the same
 454         length. The length of these lists must be the same as the length of the
 455         header.
 456     :param out_file_name: The name (relative or full path) where the
 457         generated html table is written.
 458     :param legend: The legend to display below the table.
 459     :param footnote: The footnote to display below the table (and legend).
 460     :param sort_data: If True the data sorting is enabled.
 461     :param title: The table (and file) title.
 462     :param generate_rst: If True, wrapping rst file is generated.
 463     :type header: list
 464     :type data: list of lists
 465     :type out_file_name: str
 466     :type legend: str
 467     :type footnote: str
 468     :type sort_data: bool
 469     :type title: str
 470     :type generate_rst: bool
 471     """
 472
 473     try:
 474         idx = header.index(u"Test Case")
 475     except ValueError:
 476         idx = 0
 477     params = {
 478         u"align-hdr": (
 479             [u"left", u"right"],
 480             [u"left", u"left", u"right"],
 481             [u"left", u"left", u"left", u"right"]
 482         ),
 483         u"align-itm": (
 484             [u"left", u"right"],
 485             [u"left", u"left", u"right"],
 486             [u"left", u"left", u"left", u"right"]
 487         ),
 488         u"width": ([15, 9], [4, 24, 10], [4, 4, 32, 10])
 489     }
 490
 491     df_data = pd.DataFrame(data, columns=header)
 492
 493     if sort_data:
 494         df_sorted = [df_data.sort_values(
 495             by=[key, header[idx]], ascending=[True, True]
 496             if key != header[idx] else [False, True]) for key in header]
 497         df_sorted_rev = [df_data.sort_values(
 498             by=[key, header[idx]], ascending=[False, True]
 499             if key != header[idx] else [True, True]) for key in header]
 500         df_sorted.extend(df_sorted_rev)
 501     else:
 502         df_sorted = df_data
 503
 504     fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
 505                    for idx in range(len(df_data))]]
 506     table_header = dict(
 507         values=[f"<b>{item.replace(u',', u',<br>')}</b>" for item in header],
 508         fill_color=u"#7eade7",
 509         align=params[u"align-hdr"][idx],
 510         font=dict(
 511             family=u"Courier New",
 512             size=12
 513         )
 514     )
 515
 516     fig = go.Figure()
 517
 518     if sort_data:
 519         for table in df_sorted:
 520             columns = [table.get(col) for col in header]
 521             fig.add_trace(
 522                 go.Table(
 523                     columnwidth=params[u"width"][idx],
 524                     header=table_header,
 525                     cells=dict(
 526                         values=columns,
 527                         fill_color=fill_color,
 528                         align=params[u"align-itm"][idx],
 529                         font=dict(
 530                             family=u"Courier New",
 531                             size=12
 532                         )
 533                     )
 534                 )
 535             )
 536
 537         buttons = list()
 538         menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
 539         menu_items.extend([f"<b>{itm}</b> (descending)" for itm in header])
 540         for idx, hdr in enumerate(menu_items):
 541             visible = [False, ] * len(menu_items)
 542             visible[idx] = True
 543             buttons.append(
 544                 dict(
 545                     label=hdr.replace(u" [Mpps]", u""),
 546                     method=u"update",
 547                     args=[{u"visible": visible}],
 548                 )
 549             )
 550
 551         fig.update_layout(
 552             updatemenus=[
 553                 go.layout.Updatemenu(
 554                     type=u"dropdown",
 555                     direction=u"down",
 556                     x=0.0,
 557                     xanchor=u"left",
 558                     y=1.002,
 559                     yanchor=u"bottom",
 560                     active=len(menu_items) - 1,
 561                     buttons=list(buttons)
 562                 )
 563             ],
 564         )
 565     else:
 566         fig.add_trace(
 567             go.Table(
 568                 columnwidth=params[u"width"][idx],
 569                 header=table_header,
 570                 cells=dict(
 571                     values=[df_sorted.get(col) for col in header],
 572                     fill_color=fill_color,
 573                     align=params[u"align-itm"][idx],
 574                     font=dict(
 575                         family=u"Courier New",
 576                         size=12
 577                     )
 578                 )
 579             )
 580         )
 581
 582     ploff.plot(
 583         fig,
 584         show_link=False,
 585         auto_open=False,
 586         filename=f"{out_file_name}_in.html"
 587     )
 588
 589     if not generate_rst:
 590         return
 591
 592     file_name = out_file_name.split(u"/")[-1]
 593     if u"vpp" in out_file_name:
 594         path = u"_tmp/src/vpp_performance_tests/comparisons/"
 595     else:
 596         path = u"_tmp/src/dpdk_performance_tests/comparisons/"
 597     with open(f"{path}{file_name}.rst", u"wt") as rst_file:
 598         rst_file.write(
 599             u"\n"
 600             u".. |br| raw:: html\n\n    <br />\n\n\n"
 601             u".. |prein| raw:: html\n\n    <pre>\n\n\n"
 602             u".. |preout| raw:: html\n\n    </pre>\n\n"
 603         )
 604         if title:
 605             rst_file.write(f"{title}\n")
 606             rst_file.write(f"{u'`' * len(title)}\n\n")
 607         rst_file.write(
 608             u".. raw:: html\n\n"
 609             f'    <iframe frameborder="0" scrolling="no" '
 610             f'width="1600" height="1200" '
 611             f'src="../..{out_file_name.replace(u"_build", u"")}_in.html">'
 612             f'</iframe>\n\n'
 613         )
 614
 615         # TODO: Use html (rst) list for legend and footnote
 616         if legend:
 617             rst_file.write(legend[1:].replace(u"\n", u" |br| "))
 618         if footnote:
 619             rst_file.write(footnote.replace(u"\n", u" |br| ")[1:])
 620
 621
 622 def table_soak_vs_ndr(table, input_data):
 623     """Generate the table(s) with algorithm: table_soak_vs_ndr
 624     specified in the specification file.
 625
 626     :param table: Table to generate.
 627     :param input_data: Data to process.
 628     :type table: pandas.Series
 629     :type input_data: InputData
 630     """
 631
 632     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 633
 634     # Transform the data
 635     logging.info(
 636         f"    Creating the data set for the {table.get(u'type', u'')} "
 637         f"{table.get(u'title', u'')}."
 638     )
 639     data = input_data.filter_data(table, continue_on_error=True)
 640
 641     # Prepare the header of the table
 642     try:
 643         header = [
 644             u"Test Case",
 645             f"Avg({table[u'reference'][u'title']})",
 646             f"Stdev({table[u'reference'][u'title']})",
 647             f"Avg({table[u'compare'][u'title']})",
 648             f"Stdev{table[u'compare'][u'title']})",
 649             u"Diff",
 650             u"Stdev(Diff)"
 651         ]
 652         header_str = u";".join(header) + u"\n"
 653         legend = (
 654             u"\nLegend:\n"
 655             f"Avg({table[u'reference'][u'title']}): "
 656             f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
 657             f"from a series of runs of the listed tests.\n"
 658             f"Stdev({table[u'reference'][u'title']}): "
 659             f"Standard deviation value of {table[u'reference'][u'title']} "
 660             f"[Mpps] computed from a series of runs of the listed tests.\n"
 661             f"Avg({table[u'compare'][u'title']}): "
 662             f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
 663             f"a series of runs of the listed tests.\n"
 664             f"Stdev({table[u'compare'][u'title']}): "
 665             f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
 666             f"computed from a series of runs of the listed tests.\n"
 667             f"Diff({table[u'reference'][u'title']},"
 668             f"{table[u'compare'][u'title']}): "
 669             f"Percentage change calculated for mean values.\n"
 670             u"Stdev(Diff): "
 671             u"Standard deviation of percentage change calculated for mean "
 672             u"values."
 673         )
 674     except (AttributeError, KeyError) as err:
 675         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 676         return
 677
 678     # Create a list of available SOAK test results:
 679     tbl_dict = dict()
 680     for job, builds in table[u"compare"][u"data"].items():
 681         for build in builds:
 682             for tst_name, tst_data in data[job][str(build)].items():
 683                 if tst_data[u"type"] == u"SOAK":
 684                     tst_name_mod = tst_name.replace(u"-soak", u"")
 685                     if tbl_dict.get(tst_name_mod, None) is None:
 686                         groups = re.search(REGEX_NIC, tst_data[u"parent"])
 687                         nic = groups.group(0) if groups else u""
 688                         name = (
 689                             f"{nic}-"
 690                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 691                         )
 692                         tbl_dict[tst_name_mod] = {
 693                             u"name": name,
 694                             u"ref-data": list(),
 695                             u"cmp-data": list()
 696                         }
 697                     try:
 698                         tbl_dict[tst_name_mod][u"cmp-data"].append(
 699                             tst_data[u"throughput"][u"LOWER"])
 700                     except (KeyError, TypeError):
 701                         pass
 702     tests_lst = tbl_dict.keys()
 703
 704     # Add corresponding NDR test results:
 705     for job, builds in table[u"reference"][u"data"].items():
 706         for build in builds:
 707             for tst_name, tst_data in data[job][str(build)].items():
 708                 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
 709                     replace(u"-mrr", u"")
 710                 if tst_name_mod not in tests_lst:
 711                     continue
 712                 try:
 713                     if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
 714                         continue
 715                     if table[u"include-tests"] == u"MRR":
 716                         result = (tst_data[u"result"][u"receive-rate"],
 717                                   tst_data[u"result"][u"receive-stdev"])
 718                     elif table[u"include-tests"] == u"PDR":
 719                         result = \
 720                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
 721                     elif table[u"include-tests"] == u"NDR":
 722                         result = \
 723                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
 724                     else:
 725                         result = None
 726                     if result is not None:
 727                         tbl_dict[tst_name_mod][u"ref-data"].append(
 728                             result)
 729                 except (KeyError, TypeError):
 730                     continue
 731
 732     tbl_lst = list()
 733     for tst_name in tbl_dict:
 734         item = [tbl_dict[tst_name][u"name"], ]
 735         data_r = tbl_dict[tst_name][u"ref-data"]
 736         if data_r:
 737             if table[u"include-tests"] == u"MRR":
 738                 data_r_mean = data_r[0][0]
 739                 data_r_stdev = data_r[0][1]
 740             else:
 741                 data_r_mean = mean(data_r)
 742                 data_r_stdev = stdev(data_r)
 743             item.append(round(data_r_mean / 1e6, 1))
 744             item.append(round(data_r_stdev / 1e6, 1))
 745         else:
 746             data_r_mean = None
 747             data_r_stdev = None
 748             item.extend([None, None])
 749         data_c = tbl_dict[tst_name][u"cmp-data"]
 750         if data_c:
 751             if table[u"include-tests"] == u"MRR":
 752                 data_c_mean = data_c[0][0]
 753                 data_c_stdev = data_c[0][1]
 754             else:
 755                 data_c_mean = mean(data_c)
 756                 data_c_stdev = stdev(data_c)
 757             item.append(round(data_c_mean / 1e6, 1))
 758             item.append(round(data_c_stdev / 1e6, 1))
 759         else:
 760             data_c_mean = None
 761             data_c_stdev = None
 762             item.extend([None, None])
 763         if data_r_mean is not None and data_c_mean is not None:
 764             delta, d_stdev = relative_change_stdev(
 765                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
 766             try:
 767                 item.append(round(delta))
 768             except ValueError:
 769                 item.append(delta)
 770             try:
 771                 item.append(round(d_stdev))
 772             except ValueError:
 773                 item.append(d_stdev)
 774             tbl_lst.append(item)
 775
 776     # Sort the table according to the relative change
 777     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 778
 779     # Generate csv tables:
 780     csv_file = f"{table[u'output-file']}.csv"
 781     with open(csv_file, u"wt") as file_handler:
 782         file_handler.write(header_str)
 783         for test in tbl_lst:
 784             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 785
 786     convert_csv_to_pretty_txt(
 787         csv_file, f"{table[u'output-file']}.txt", delimiter=u";"
 788     )
 789     with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
 790         txt_file.write(legend)
 791
 792     # Generate html table:
 793     _tpc_generate_html_table(
 794         header,
 795         tbl_lst,
 796         table[u'output-file'],
 797         legend=legend,
 798         title=table.get(u"title", u"")
 799     )
 800
 801
 802 def table_perf_trending_dash(table, input_data):
 803     """Generate the table(s) with algorithm:
 804     table_perf_trending_dash
 805     specified in the specification file.
 806
 807     :param table: Table to generate.
 808     :param input_data: Data to process.
 809     :type table: pandas.Series
 810     :type input_data: InputData
 811     """
 812
 813     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 814
 815     # Transform the data
 816     logging.info(
 817         f"    Creating the data set for the {table.get(u'type', u'')} "
 818         f"{table.get(u'title', u'')}."
 819     )
 820     data = input_data.filter_data(table, continue_on_error=True)
 821
 822     # Prepare the header of the tables
 823     header = [
 824         u"Test Case",
 825         u"Trend [Mpps]",
 826         u"Short-Term Change [%]",
 827         u"Long-Term Change [%]",
 828         u"Regressions [#]",
 829         u"Progressions [#]"
 830     ]
 831     header_str = u",".join(header) + u"\n"
 832
 833     incl_tests = table.get(u"include-tests", u"MRR")
 834
 835     # Prepare data to the table:
 836     tbl_dict = dict()
 837     for job, builds in table[u"data"].items():
 838         for build in builds:
 839             for tst_name, tst_data in data[job][str(build)].items():
 840                 if tst_name.lower() in table.get(u"ignore-list", list()):
 841                     continue
 842                 if tbl_dict.get(tst_name, None) is None:
 843                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
 844                     if not groups:
 845                         continue
 846                     nic = groups.group(0)
 847                     tbl_dict[tst_name] = {
 848                         u"name": f"{nic}-{tst_data[u'name']}",
 849                         u"data": OrderedDict()
 850                     }
 851                 try:
 852                     if incl_tests == u"MRR":
 853                         tbl_dict[tst_name][u"data"][str(build)] = \
 854                             tst_data[u"result"][u"receive-rate"]
 855                     elif incl_tests == u"NDR":
 856                         tbl_dict[tst_name][u"data"][str(build)] = \
 857                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
 858                     elif incl_tests == u"PDR":
 859                         tbl_dict[tst_name][u"data"][str(build)] = \
 860                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
 861                 except (TypeError, KeyError):
 862                     pass  # No data in output.xml for this test
 863
 864     tbl_lst = list()
 865     for tst_name in tbl_dict:
 866         data_t = tbl_dict[tst_name][u"data"]
 867         if len(data_t) < 2:
 868             continue
 869
 870         classification_lst, avgs, _ = classify_anomalies(data_t)
 871
 872         win_size = min(len(data_t), table[u"window"])
 873         long_win_size = min(len(data_t), table[u"long-trend-window"])
 874
 875         try:
 876             max_long_avg = max(
 877                 [x for x in avgs[-long_win_size:-win_size]
 878                  if not isnan(x)])
 879         except ValueError:
 880             max_long_avg = nan
 881         last_avg = avgs[-1]
 882         avg_week_ago = avgs[max(-win_size, -len(avgs))]
 883
 884         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
 885             rel_change_last = nan
 886         else:
 887             rel_change_last = round(
 888                 ((last_avg - avg_week_ago) / avg_week_ago) * 1e2, 2)
 889
 890         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
 891             rel_change_long = nan
 892         else:
 893             rel_change_long = round(
 894                 ((last_avg - max_long_avg) / max_long_avg) * 1e2, 2)
 895
 896         if classification_lst:
 897             if isnan(rel_change_last) and isnan(rel_change_long):
 898                 continue
 899             if isnan(last_avg) or isnan(rel_change_last) or \
 900                     isnan(rel_change_long):
 901                 continue
 902             tbl_lst.append(
 903                 [tbl_dict[tst_name][u"name"],
 904                  round(last_avg / 1e6, 2),
 905                  rel_change_last,
 906                  rel_change_long,
 907                  classification_lst[-win_size+1:].count(u"regression"),
 908                  classification_lst[-win_size+1:].count(u"progression")])
 909
 910     tbl_lst.sort(key=lambda rel: rel[0])
 911     tbl_lst.sort(key=lambda rel: rel[3])
 912     tbl_lst.sort(key=lambda rel: rel[2])
 913
 914     tbl_sorted = list()
 915     for nrr in range(table[u"window"], -1, -1):
 916         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
 917         for nrp in range(table[u"window"], -1, -1):
 918             tbl_out = [item for item in tbl_reg if item[5] == nrp]
 919             tbl_sorted.extend(tbl_out)
 920
 921     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
 922
 923     logging.info(f"    Writing file: {file_name}")
 924     with open(file_name, u"wt") as file_handler:
 925         file_handler.write(header_str)
 926         for test in tbl_sorted:
 927             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
 928
 929     logging.info(f"    Writing file: {table[u'output-file']}.txt")
 930     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
 931
 932
 933 def _generate_url(testbed, test_name):
 934     """Generate URL to a trending plot from the name of the test case.
 935
 936     :param testbed: The testbed used for testing.
 937     :param test_name: The name of the test case.
 938     :type testbed: str
 939     :type test_name: str
 940     :returns: The URL to the plot with the trending data for the given test
 941         case.
 942     :rtype str
 943     """
 944
 945     if u"x520" in test_name:
 946         nic = u"x520"
 947     elif u"x710" in test_name:
 948         nic = u"x710"
 949     elif u"xl710" in test_name:
 950         nic = u"xl710"
 951     elif u"xxv710" in test_name:
 952         nic = u"xxv710"
 953     elif u"vic1227" in test_name:
 954         nic = u"vic1227"
 955     elif u"vic1385" in test_name:
 956         nic = u"vic1385"
 957     elif u"x553" in test_name:
 958         nic = u"x553"
 959     elif u"cx556" in test_name or u"cx556a" in test_name:
 960         nic = u"cx556a"
 961     else:
 962         nic = u""
 963
 964     if u"64b" in test_name:
 965         frame_size = u"64b"
 966     elif u"78b" in test_name:
 967         frame_size = u"78b"
 968     elif u"imix" in test_name:
 969         frame_size = u"imix"
 970     elif u"9000b" in test_name:
 971         frame_size = u"9000b"
 972     elif u"1518b" in test_name:
 973         frame_size = u"1518b"
 974     elif u"114b" in test_name:
 975         frame_size = u"114b"
 976     else:
 977         frame_size = u""
 978
 979     if u"1t1c" in test_name or \
 980         (u"-1c-" in test_name and
 981          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 982         cores = u"1t1c"
 983     elif u"2t2c" in test_name or \
 984          (u"-2c-" in test_name and
 985           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 986         cores = u"2t2c"
 987     elif u"4t4c" in test_name or \
 988          (u"-4c-" in test_name and
 989           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 990         cores = u"4t4c"
 991     elif u"2t1c" in test_name or \
 992          (u"-1c-" in test_name and
 993           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
 994         cores = u"2t1c"
 995     elif u"4t2c" in test_name or \
 996          (u"-2c-" in test_name and
 997           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
 998         cores = u"4t2c"
 999     elif u"8t4c" in test_name or \
1000          (u"-4c-" in test_name and
1001           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1002         cores = u"8t4c"
1003     else:
1004         cores = u""
1005
1006     if u"testpmd" in test_name:
1007         driver = u"testpmd"
1008     elif u"l3fwd" in test_name:
1009         driver = u"l3fwd"
1010     elif u"avf" in test_name:
1011         driver = u"avf"
1012     elif u"rdma" in test_name:
1013         driver = u"rdma"
1014     elif u"dnv" in testbed or u"tsh" in testbed:
1015         driver = u"ixgbe"
1016     else:
1017         driver = u"dpdk"
1018
1019     if u"macip-iacl1s" in test_name:
1020         bsf = u"features-macip-iacl1"
1021     elif u"macip-iacl10s" in test_name:
1022         bsf = u"features-macip-iacl01"
1023     elif u"macip-iacl50s" in test_name:
1024         bsf = u"features-macip-iacl50"
1025     elif u"iacl1s" in test_name:
1026         bsf = u"features-iacl1"
1027     elif u"iacl10s" in test_name:
1028         bsf = u"features-iacl10"
1029     elif u"iacl50s" in test_name:
1030         bsf = u"features-iacl50"
1031     elif u"oacl1s" in test_name:
1032         bsf = u"features-oacl1"
1033     elif u"oacl10s" in test_name:
1034         bsf = u"features-oacl10"
1035     elif u"oacl50s" in test_name:
1036         bsf = u"features-oacl50"
1037     elif u"udpsrcscale" in test_name:
1038         bsf = u"features-udp"
1039     elif u"iacl" in test_name:
1040         bsf = u"features"
1041     elif u"policer" in test_name:
1042         bsf = u"features"
1043     elif u"cop" in test_name:
1044         bsf = u"features"
1045     elif u"nat" in test_name:
1046         bsf = u"features"
1047     elif u"macip" in test_name:
1048         bsf = u"features"
1049     elif u"scale" in test_name:
1050         bsf = u"scale"
1051     elif u"base" in test_name:
1052         bsf = u"base"
1053     else:
1054         bsf = u"base"
1055
1056     if u"114b" in test_name and u"vhost" in test_name:
1057         domain = u"vts"
1058     elif u"testpmd" in test_name or u"l3fwd" in test_name:
1059         domain = u"dpdk"
1060     elif u"memif" in test_name:
1061         domain = u"container_memif"
1062     elif u"srv6" in test_name:
1063         domain = u"srv6"
1064     elif u"vhost" in test_name:
1065         domain = u"vhost"
1066         if u"vppl2xc" in test_name:
1067             driver += u"-vpp"
1068         else:
1069             driver += u"-testpmd"
1070         if u"lbvpplacp" in test_name:
1071             bsf += u"-link-bonding"
1072     elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1073         domain = u"nf_service_density_vnfc"
1074     elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1075         domain = u"nf_service_density_cnfc"
1076     elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1077         domain = u"nf_service_density_cnfp"
1078     elif u"ipsec" in test_name:
1079         domain = u"ipsec"
1080         if u"sw" in test_name:
1081             bsf += u"-sw"
1082         elif u"hw" in test_name:
1083             bsf += u"-hw"
1084     elif u"ethip4vxlan" in test_name:
1085         domain = u"ip4_tunnels"
1086     elif u"ip4base" in test_name or u"ip4scale" in test_name:
1087         domain = u"ip4"
1088     elif u"ip6base" in test_name or u"ip6scale" in test_name:
1089         domain = u"ip6"
1090     elif u"l2xcbase" in test_name or \
1091             u"l2xcscale" in test_name or \
1092             u"l2bdbasemaclrn" in test_name or \
1093             u"l2bdscale" in test_name or \
1094             u"l2patch" in test_name:
1095         domain = u"l2"
1096     else:
1097         domain = u""
1098
1099     file_name = u"-".join((domain, testbed, nic)) + u".html#"
1100     anchor_name = u"-".join((frame_size, cores, bsf, driver))
1101
1102     return file_name + anchor_name
1103
1104
1105 def table_perf_trending_dash_html(table, input_data):
1106     """Generate the table(s) with algorithm:
1107     table_perf_trending_dash_html specified in the specification
1108     file.
1109
1110     :param table: Table to generate.
1111     :param input_data: Data to process.
1112     :type table: dict
1113     :type input_data: InputData
1114     """
1115
1116     _ = input_data
1117
1118     if not table.get(u"testbed", None):
1119         logging.error(
1120             f"The testbed is not defined for the table "
1121             f"{table.get(u'title', u'')}. Skipping."
1122         )
1123         return
1124
1125     test_type = table.get(u"test-type", u"MRR")
1126     if test_type not in (u"MRR", u"NDR", u"PDR"):
1127         logging.error(
1128             f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
1129             f"Skipping."
1130         )
1131         return
1132
1133     if test_type in (u"NDR", u"PDR"):
1134         lnk_dir = u"../ndrpdr_trending/"
1135         lnk_sufix = f"-{test_type.lower()}"
1136     else:
1137         lnk_dir = u"../trending/"
1138         lnk_sufix = u""
1139
1140     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1141
1142     try:
1143         with open(table[u"input-file"], u'rt') as csv_file:
1144             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1145     except KeyError:
1146         logging.warning(u"The input file is not defined.")
1147         return
1148     except csv.Error as err:
1149         logging.warning(
1150             f"Not possible to process the file {table[u'input-file']}.\n"
1151             f"{repr(err)}"
1152         )
1153         return
1154
1155     # Table:
1156     dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1157
1158     # Table header:
1159     trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1160     for idx, item in enumerate(csv_lst[0]):
1161         alignment = u"left" if idx == 0 else u"center"
1162         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1163         thead.text = item
1164
1165     # Rows:
1166     colors = {
1167         u"regression": (
1168             u"#ffcccc",
1169             u"#ff9999"
1170         ),
1171         u"progression": (
1172             u"#c6ecc6",
1173             u"#9fdf9f"
1174         ),
1175         u"normal": (
1176             u"#e9f1fb",
1177             u"#d4e4f7"
1178         )
1179     }
1180     for r_idx, row in enumerate(csv_lst[1:]):
1181         if int(row[4]):
1182             color = u"regression"
1183         elif int(row[5]):
1184             color = u"progression"
1185         else:
1186             color = u"normal"
1187         trow = ET.SubElement(
1188             dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
1189         )
1190
1191         # Columns:
1192         for c_idx, item in enumerate(row):
1193             tdata = ET.SubElement(
1194                 trow,
1195                 u"td",
1196                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1197             )
1198             # Name:
1199             if c_idx == 0 and table.get(u"add-links", True):
1200                 ref = ET.SubElement(
1201                     tdata,
1202                     u"a",
1203                     attrib=dict(
1204                         href=f"{lnk_dir}"
1205                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1206                              f"{lnk_sufix}"
1207                     )
1208                 )
1209                 ref.text = item
1210             else:
1211                 tdata.text = item
1212     try:
1213         with open(table[u"output-file"], u'w') as html_file:
1214             logging.info(f"    Writing file: {table[u'output-file']}")
1215             html_file.write(u".. raw:: html\n\n\t")
1216             html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
1217             html_file.write(u"\n\t<p><br><br></p>\n")
1218     except KeyError:
1219         logging.warning(u"The output file is not defined.")
1220         return
1221
1222
1223 def table_last_failed_tests(table, input_data):
1224     """Generate the table(s) with algorithm: table_last_failed_tests
1225     specified in the specification file.
1226
1227     :param table: Table to generate.
1228     :param input_data: Data to process.
1229     :type table: pandas.Series
1230     :type input_data: InputData
1231     """
1232
1233     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1234
1235     # Transform the data
1236     logging.info(
1237         f"    Creating the data set for the {table.get(u'type', u'')} "
1238         f"{table.get(u'title', u'')}."
1239     )
1240
1241     data = input_data.filter_data(table, continue_on_error=True)
1242
1243     if data is None or data.empty:
1244         logging.warning(
1245             f"    No data for the {table.get(u'type', u'')} "
1246             f"{table.get(u'title', u'')}."
1247         )
1248         return
1249
1250     tbl_list = list()
1251     for job, builds in table[u"data"].items():
1252         for build in builds:
1253             build = str(build)
1254             try:
1255                 version = input_data.metadata(job, build).get(u"version", u"")
1256             except KeyError:
1257                 logging.error(f"Data for {job}: {build} is not present.")
1258                 return
1259             tbl_list.append(build)
1260             tbl_list.append(version)
1261             failed_tests = list()
1262             passed = 0
1263             failed = 0
1264             for tst_data in data[job][build].values:
1265                 if tst_data[u"status"] != u"FAIL":
1266                     passed += 1
1267                     continue
1268                 failed += 1
1269                 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1270                 if not groups:
1271                     continue
1272                 nic = groups.group(0)
1273                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
1274             tbl_list.append(str(passed))
1275             tbl_list.append(str(failed))
1276             tbl_list.extend(failed_tests)
1277
1278     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1279     logging.info(f"    Writing file: {file_name}")
1280     with open(file_name, u"wt") as file_handler:
1281         for test in tbl_list:
1282             file_handler.write(test + u'\n')
1283
1284
1285 def table_failed_tests(table, input_data):
1286     """Generate the table(s) with algorithm: table_failed_tests
1287     specified in the specification file.
1288
1289     :param table: Table to generate.
1290     :param input_data: Data to process.
1291     :type table: pandas.Series
1292     :type input_data: InputData
1293     """
1294
1295     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1296
1297     # Transform the data
1298     logging.info(
1299         f"    Creating the data set for the {table.get(u'type', u'')} "
1300         f"{table.get(u'title', u'')}."
1301     )
1302     data = input_data.filter_data(table, continue_on_error=True)
1303
1304     test_type = u"MRR"
1305     if u"NDRPDR" in table.get(u"filter", list()):
1306         test_type = u"NDRPDR"
1307
1308     # Prepare the header of the tables
1309     header = [
1310         u"Test Case",
1311         u"Failures [#]",
1312         u"Last Failure [Time]",
1313         u"Last Failure [VPP-Build-Id]",
1314         u"Last Failure [CSIT-Job-Build-Id]"
1315     ]
1316
1317     # Generate the data for the table according to the model in the table
1318     # specification
1319
1320     now = dt.utcnow()
1321     timeperiod = timedelta(int(table.get(u"window", 7)))
1322
1323     tbl_dict = dict()
1324     for job, builds in table[u"data"].items():
1325         for build in builds:
1326             build = str(build)
1327             for tst_name, tst_data in data[job][build].items():
1328                 if tst_name.lower() in table.get(u"ignore-list", list()):
1329                     continue
1330                 if tbl_dict.get(tst_name, None) is None:
1331                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1332                     if not groups:
1333                         continue
1334                     nic = groups.group(0)
1335                     tbl_dict[tst_name] = {
1336                         u"name": f"{nic}-{tst_data[u'name']}",
1337                         u"data": OrderedDict()
1338                     }
1339                 try:
1340                     generated = input_data.metadata(job, build).\
1341                         get(u"generated", u"")
1342                     if not generated:
1343                         continue
1344                     then = dt.strptime(generated, u"%Y%m%d %H:%M")
1345                     if (now - then) <= timeperiod:
1346                         tbl_dict[tst_name][u"data"][build] = (
1347                             tst_data[u"status"],
1348                             generated,
1349                             input_data.metadata(job, build).get(u"version",
1350                                                                 u""),
1351                             build
1352                         )
1353                 except (TypeError, KeyError) as err:
1354                     logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
1355
1356     max_fails = 0
1357     tbl_lst = list()
1358     for tst_data in tbl_dict.values():
1359         fails_nr = 0
1360         fails_last_date = u""
1361         fails_last_vpp = u""
1362         fails_last_csit = u""
1363         for val in tst_data[u"data"].values():
1364             if val[0] == u"FAIL":
1365                 fails_nr += 1
1366                 fails_last_date = val[1]
1367                 fails_last_vpp = val[2]
1368                 fails_last_csit = val[3]
1369         if fails_nr:
1370             max_fails = fails_nr if fails_nr > max_fails else max_fails
1371             tbl_lst.append([
1372                 tst_data[u"name"],
1373                 fails_nr,
1374                 fails_last_date,
1375                 fails_last_vpp,
1376                 f"{u'mrr-daily' if test_type == u'MRR' else u'ndrpdr-weekly'}"
1377                 f"-build-{fails_last_csit}"
1378             ])
1379
1380     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1381     tbl_sorted = list()
1382     for nrf in range(max_fails, -1, -1):
1383         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1384         tbl_sorted.extend(tbl_fails)
1385
1386     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1387     logging.info(f"    Writing file: {file_name}")
1388     with open(file_name, u"wt") as file_handler:
1389         file_handler.write(u",".join(header) + u"\n")
1390         for test in tbl_sorted:
1391             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1392
1393     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1394     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1395
1396
1397 def table_failed_tests_html(table, input_data):
1398     """Generate the table(s) with algorithm: table_failed_tests_html
1399     specified in the specification file.
1400
1401     :param table: Table to generate.
1402     :param input_data: Data to process.
1403     :type table: pandas.Series
1404     :type input_data: InputData
1405     """
1406
1407     _ = input_data
1408
1409     if not table.get(u"testbed", None):
1410         logging.error(
1411             f"The testbed is not defined for the table "
1412             f"{table.get(u'title', u'')}. Skipping."
1413         )
1414         return
1415
1416     test_type = table.get(u"test-type", u"MRR")
1417     if test_type not in (u"MRR", u"NDR", u"PDR", u"NDRPDR"):
1418         logging.error(
1419             f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
1420             f"Skipping."
1421         )
1422         return
1423
1424     if test_type in (u"NDRPDR", u"NDR", u"PDR"):
1425         lnk_dir = u"../ndrpdr_trending/"
1426         lnk_sufix = u"-pdr"
1427     else:
1428         lnk_dir = u"../trending/"
1429         lnk_sufix = u""
1430
1431     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1432
1433     try:
1434         with open(table[u"input-file"], u'rt') as csv_file:
1435             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1436     except KeyError:
1437         logging.warning(u"The input file is not defined.")
1438         return
1439     except csv.Error as err:
1440         logging.warning(
1441             f"Not possible to process the file {table[u'input-file']}.\n"
1442             f"{repr(err)}"
1443         )
1444         return
1445
1446     # Table:
1447     failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1448
1449     # Table header:
1450     trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1451     for idx, item in enumerate(csv_lst[0]):
1452         alignment = u"left" if idx == 0 else u"center"
1453         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1454         thead.text = item
1455
1456     # Rows:
1457     colors = (u"#e9f1fb", u"#d4e4f7")
1458     for r_idx, row in enumerate(csv_lst[1:]):
1459         background = colors[r_idx % 2]
1460         trow = ET.SubElement(
1461             failed_tests, u"tr", attrib=dict(bgcolor=background)
1462         )
1463
1464         # Columns:
1465         for c_idx, item in enumerate(row):
1466             tdata = ET.SubElement(
1467                 trow,
1468                 u"td",
1469                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1470             )
1471             # Name:
1472             if c_idx == 0 and table.get(u"add-links", True):
1473                 ref = ET.SubElement(
1474                     tdata,
1475                     u"a",
1476                     attrib=dict(
1477                         href=f"{lnk_dir}"
1478                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1479                              f"{lnk_sufix}"
1480                     )
1481                 )
1482                 ref.text = item
1483             else:
1484                 tdata.text = item
1485     try:
1486         with open(table[u"output-file"], u'w') as html_file:
1487             logging.info(f"    Writing file: {table[u'output-file']}")
1488             html_file.write(u".. raw:: html\n\n\t")
1489             html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
1490             html_file.write(u"\n\t<p><br><br></p>\n")
1491     except KeyError:
1492         logging.warning(u"The output file is not defined.")
1493         return
1494
1495
1496 def table_comparison(table, input_data):
1497     """Generate the table(s) with algorithm: table_comparison
1498     specified in the specification file.
1499
1500     :param table: Table to generate.
1501     :param input_data: Data to process.
1502     :type table: pandas.Series
1503     :type input_data: InputData
1504     """
1505     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1506
1507     # Transform the data
1508     logging.info(
1509         f"    Creating the data set for the {table.get(u'type', u'')} "
1510         f"{table.get(u'title', u'')}."
1511     )
1512
1513     columns = table.get(u"columns", None)
1514     if not columns:
1515         logging.error(
1516             f"No columns specified for {table.get(u'title', u'')}. Skipping."
1517         )
1518         return
1519
1520     cols = list()
1521     for idx, col in enumerate(columns):
1522         if col.get(u"data-set", None) is None:
1523             logging.warning(f"No data for column {col.get(u'title', u'')}")
1524             continue
1525         tag = col.get(u"tag", None)
1526         data = input_data.filter_data(
1527             table,
1528             params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1529             data=col[u"data-set"],
1530             continue_on_error=True
1531         )
1532         col_data = {
1533             u"title": col.get(u"title", f"Column{idx}"),
1534             u"data": dict()
1535         }
1536         for builds in data.values:
1537             for build in builds:
1538                 for tst_name, tst_data in build.items():
1539                     if tag and tag not in tst_data[u"tags"]:
1540                         continue
1541                     tst_name_mod = \
1542                         _tpc_modify_test_name(tst_name, ignore_nic=True).\
1543                         replace(u"2n1l-", u"")
1544                     if col_data[u"data"].get(tst_name_mod, None) is None:
1545                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
1546                         if u"across testbeds" in table[u"title"].lower() or \
1547                                 u"across topologies" in table[u"title"].lower():
1548                             name = _tpc_modify_displayed_test_name(name)
1549                         col_data[u"data"][tst_name_mod] = {
1550                             u"name": name,
1551                             u"replace": True,
1552                             u"data": list(),
1553                             u"mean": None,
1554                             u"stdev": None
1555                         }
1556                     _tpc_insert_data(
1557                         target=col_data[u"data"][tst_name_mod],
1558                         src=tst_data,
1559                         include_tests=table[u"include-tests"]
1560                     )
1561
1562         replacement = col.get(u"data-replacement", None)
1563         if replacement:
1564             rpl_data = input_data.filter_data(
1565                 table,
1566                 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1567                 data=replacement,
1568                 continue_on_error=True
1569             )
1570             for builds in rpl_data.values:
1571                 for build in builds:
1572                     for tst_name, tst_data in build.items():
1573                         if tag and tag not in tst_data[u"tags"]:
1574                             continue
1575                         tst_name_mod = \
1576                             _tpc_modify_test_name(tst_name, ignore_nic=True).\
1577                             replace(u"2n1l-", u"")
1578                         if col_data[u"data"].get(tst_name_mod, None) is None:
1579                             name = tst_data[u'name'].rsplit(u'-', 1)[0]
1580                             if u"across testbeds" in table[u"title"].lower() \
1581                                     or u"across topologies" in \
1582                                     table[u"title"].lower():
1583                                 name = _tpc_modify_displayed_test_name(name)
1584                             col_data[u"data"][tst_name_mod] = {
1585                                 u"name": name,
1586                                 u"replace": False,
1587                                 u"data": list(),
1588                                 u"mean": None,
1589                                 u"stdev": None
1590                             }
1591                         if col_data[u"data"][tst_name_mod][u"replace"]:
1592                             col_data[u"data"][tst_name_mod][u"replace"] = False
1593                             col_data[u"data"][tst_name_mod][u"data"] = list()
1594                         _tpc_insert_data(
1595                             target=col_data[u"data"][tst_name_mod],
1596                             src=tst_data,
1597                             include_tests=table[u"include-tests"]
1598                         )
1599
1600         if table[u"include-tests"] in (u"NDR", u"PDR"):
1601             for tst_name, tst_data in col_data[u"data"].items():
1602                 if tst_data[u"data"]:
1603                     tst_data[u"mean"] = mean(tst_data[u"data"])
1604                     tst_data[u"stdev"] = stdev(tst_data[u"data"])
1605
1606         cols.append(col_data)
1607
1608     tbl_dict = dict()
1609     for col in cols:
1610         for tst_name, tst_data in col[u"data"].items():
1611             if tbl_dict.get(tst_name, None) is None:
1612                 tbl_dict[tst_name] = {
1613                     "name": tst_data[u"name"]
1614                 }
1615             tbl_dict[tst_name][col[u"title"]] = {
1616                 u"mean": tst_data[u"mean"],
1617                 u"stdev": tst_data[u"stdev"]
1618             }
1619
1620     if not tbl_dict:
1621         logging.warning(f"No data for table {table.get(u'title', u'')}!")
1622         return
1623
1624     tbl_lst = list()
1625     for tst_data in tbl_dict.values():
1626         row = [tst_data[u"name"], ]
1627         for col in cols:
1628             row.append(tst_data.get(col[u"title"], None))
1629         tbl_lst.append(row)
1630
1631     comparisons = table.get(u"comparisons", None)
1632     if comparisons and isinstance(comparisons, list):
1633         for idx, comp in enumerate(comparisons):
1634             try:
1635                 col_ref = int(comp[u"reference"])
1636                 col_cmp = int(comp[u"compare"])
1637             except KeyError:
1638                 logging.warning(u"Comparison: No references defined! Skipping.")
1639                 comparisons.pop(idx)
1640                 continue
1641             if not (0 < col_ref <= len(cols) and
1642                     0 < col_cmp <= len(cols)) or \
1643                     col_ref == col_cmp:
1644                 logging.warning(f"Wrong values of reference={col_ref} "
1645                                 f"and/or compare={col_cmp}. Skipping.")
1646                 comparisons.pop(idx)
1647                 continue
1648
1649     tbl_cmp_lst = list()
1650     if comparisons:
1651         for row in tbl_lst:
1652             new_row = deepcopy(row)
1653             add_to_tbl = False
1654             for comp in comparisons:
1655                 ref_itm = row[int(comp[u"reference"])]
1656                 if ref_itm is None and \
1657                         comp.get(u"reference-alt", None) is not None:
1658                     ref_itm = row[int(comp[u"reference-alt"])]
1659                 cmp_itm = row[int(comp[u"compare"])]
1660                 if ref_itm is not None and cmp_itm is not None and \
1661                         ref_itm[u"mean"] is not None and \
1662                         cmp_itm[u"mean"] is not None and \
1663                         ref_itm[u"stdev"] is not None and \
1664                         cmp_itm[u"stdev"] is not None:
1665                     delta, d_stdev = relative_change_stdev(
1666                         ref_itm[u"mean"], cmp_itm[u"mean"],
1667                         ref_itm[u"stdev"], cmp_itm[u"stdev"]
1668                     )
1669                     new_row.append(
1670                         {
1671                             u"mean": delta * 1e6,
1672                             u"stdev": d_stdev * 1e6
1673                         }
1674                     )
1675                     add_to_tbl = True
1676                 else:
1677                     new_row.append(None)
1678             if add_to_tbl:
1679                 tbl_cmp_lst.append(new_row)
1680
1681     tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
1682     tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
1683
1684     rcas = list()
1685     rca_in = table.get(u"rca", None)
1686     if rca_in and isinstance(rca_in, list):
1687         for idx, itm in enumerate(rca_in):
1688             try:
1689                 with open(itm.get(u"data", u""), u"r") as rca_file:
1690                     rcas.append(
1691                         {
1692                             u"title": itm.get(u"title", f"RCA{idx}"),
1693                             u"data": load(rca_file, Loader=FullLoader)
1694                         }
1695                     )
1696             except (YAMLError, IOError) as err:
1697                 logging.warning(
1698                     f"The RCA file {itm.get(u'data', u'')} does not exist or "
1699                     f"it is corrupted!"
1700                 )
1701                 logging.debug(repr(err))
1702
1703     tbl_for_csv = list()
1704     for line in tbl_cmp_lst:
1705         row = [line[0], ]
1706         for idx, itm in enumerate(line[1:]):
1707             if itm is None:
1708                 row.append(u"NT")
1709                 row.append(u"NT")
1710             else:
1711                 row.append(round(float(itm[u'mean']) / 1e6, 3))
1712                 row.append(round(float(itm[u'stdev']) / 1e6, 3))
1713         for rca in rcas:
1714             rca_nr = rca[u"data"].get(row[0], u"-")
1715             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1716         tbl_for_csv.append(row)
1717
1718     header_csv = [u"Test Case", ]
1719     for col in cols:
1720         header_csv.append(f"Avg({col[u'title']})")
1721         header_csv.append(f"Stdev({col[u'title']})")
1722     for comp in comparisons:
1723         header_csv.append(
1724             f"Avg({comp.get(u'title', u'')})"
1725         )
1726         header_csv.append(
1727             f"Stdev({comp.get(u'title', u'')})"
1728         )
1729     header_csv.extend([rca[u"title"] for rca in rcas])
1730
1731     legend_lst = table.get(u"legend", None)
1732     if legend_lst is None:
1733         legend = u""
1734     else:
1735         legend = u"\n" + u"\n".join(legend_lst) + u"\n"
1736
1737     footnote = u""
1738     for rca in rcas:
1739         footnote += f"\n{rca[u'title']}:\n"
1740         footnote += rca[u"data"].get(u"footnote", u"")
1741
1742     csv_file = f"{table[u'output-file']}-csv.csv"
1743     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1744         file_handler.write(
1745             u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
1746         )
1747         for test in tbl_for_csv:
1748             file_handler.write(
1749                 u",".join([f'"{item}"' for item in test]) + u"\n"
1750             )
1751         if legend_lst:
1752             for item in legend_lst:
1753                 file_handler.write(f'"{item}"\n')
1754         if footnote:
1755             for itm in footnote.split(u"\n"):
1756                 file_handler.write(f'"{itm}"\n')
1757
1758     tbl_tmp = list()
1759     max_lens = [0, ] * len(tbl_cmp_lst[0])
1760     for line in tbl_cmp_lst:
1761         row = [line[0], ]
1762         for idx, itm in enumerate(line[1:]):
1763             if itm is None:
1764                 new_itm = u"NT"
1765             else:
1766                 if idx < len(cols):
1767                     new_itm = (
1768                         f"{round(float(itm[u'mean']) / 1e6, 1)} "
1769                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1770                         replace(u"nan", u"NaN")
1771                     )
1772                 else:
1773                     new_itm = (
1774                         f"{round(float(itm[u'mean']) / 1e6, 1):+} "
1775                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1776                         replace(u"nan", u"NaN")
1777                     )
1778             if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]:
1779                 max_lens[idx] = len(new_itm.rsplit(u" ", 1)[-1])
1780             row.append(new_itm)
1781
1782         tbl_tmp.append(row)
1783
1784     tbl_final = list()
1785     for line in tbl_tmp:
1786         row = [line[0], ]
1787         for idx, itm in enumerate(line[1:]):
1788             if itm in (u"NT", u"NaN"):
1789                 row.append(itm)
1790                 continue
1791             itm_lst = itm.rsplit(u"\u00B1", 1)
1792             itm_lst[-1] = \
1793                 f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
1794             row.append(u"\u00B1".join(itm_lst))
1795         for rca in rcas:
1796             rca_nr = rca[u"data"].get(row[0], u"-")
1797             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1798
1799         tbl_final.append(row)
1800
1801     header = [u"Test Case", ]
1802     header.extend([col[u"title"] for col in cols])
1803     header.extend([comp.get(u"title", u"") for comp in comparisons])
1804     header.extend([rca[u"title"] for rca in rcas])
1805
1806     # Generate csv tables:
1807     csv_file = f"{table[u'output-file']}.csv"
1808     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1809         file_handler.write(u";".join(header) + u"\n")
1810         for test in tbl_final:
1811             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1812
1813     # Generate txt table:
1814     txt_file_name = f"{table[u'output-file']}.txt"
1815     convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
1816
1817     with open(txt_file_name, u'a', encoding='utf-8') as txt_file:
1818         txt_file.write(legend)
1819         txt_file.write(footnote)
1820
1821     # Generate html table:
1822     _tpc_generate_html_table(
1823         header,
1824         tbl_final,
1825         table[u'output-file'],
1826         legend=legend,
1827         footnote=footnote,
1828         sort_data=False,
1829         title=table.get(u"title", u"")
1830     )
1831
1832
1833 def table_weekly_comparison(table, in_data):
1834     """Generate the table(s) with algorithm: table_weekly_comparison
1835     specified in the specification file.
1836
1837     :param table: Table to generate.
1838     :param in_data: Data to process.
1839     :type table: pandas.Series
1840     :type in_data: InputData
1841     """
1842     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1843
1844     # Transform the data
1845     logging.info(
1846         f"    Creating the data set for the {table.get(u'type', u'')} "
1847         f"{table.get(u'title', u'')}."
1848     )
1849
1850     incl_tests = table.get(u"include-tests", None)
1851     if incl_tests not in (u"NDR", u"PDR"):
1852         logging.error(f"Wrong tests to include specified ({incl_tests}).")
1853         return
1854
1855     nr_cols = table.get(u"nr-of-data-columns", None)
1856     if not nr_cols or nr_cols < 2:
1857         logging.error(
1858             f"No columns specified for {table.get(u'title', u'')}. Skipping."
1859         )
1860         return
1861
1862     data = in_data.filter_data(
1863         table,
1864         params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1865         continue_on_error=True
1866     )
1867
1868     header = [
1869         [u"VPP Version", ],
1870         [u"Start Timestamp", ],
1871         [u"CSIT Build", ],
1872         [u"CSIT Testbed", ]
1873     ]
1874     tbl_dict = dict()
1875     idx = 0
1876     tb_tbl = table.get(u"testbeds", None)
1877     for job_name, job_data in data.items():
1878         for build_nr, build in job_data.items():
1879             if idx >= nr_cols:
1880                 break
1881             if build.empty:
1882                 continue
1883
1884             tb_ip = in_data.metadata(job_name, build_nr).get(u"testbed", u"")
1885             if tb_ip and tb_tbl:
1886                 testbed = tb_tbl.get(tb_ip, u"")
1887             else:
1888                 testbed = u""
1889             header[2].insert(1, build_nr)
1890             header[3].insert(1, testbed)
1891             header[1].insert(
1892                 1, in_data.metadata(job_name, build_nr).get(u"generated", u"")
1893             )
1894             header[0].insert(
1895                 1, in_data.metadata(job_name, build_nr).get(u"version", u"")
1896             )
1897
1898             for tst_name, tst_data in build.items():
1899                 tst_name_mod = \
1900                     _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
1901                 if not tbl_dict.get(tst_name_mod, None):
1902                     tbl_dict[tst_name_mod] = dict(
1903                         name=tst_data[u'name'].rsplit(u'-', 1)[0],
1904                     )
1905                 try:
1906                     tbl_dict[tst_name_mod][-idx - 1] = \
1907                         tst_data[u"throughput"][incl_tests][u"LOWER"]
1908                 except (TypeError, IndexError, KeyError, ValueError):
1909                     pass
1910             idx += 1
1911
1912     if idx < nr_cols:
1913         logging.error(u"Not enough data to build the table! Skipping")
1914         return
1915
1916     cmp_dict = dict()
1917     for idx, cmp in enumerate(table.get(u"comparisons", list())):
1918         idx_ref = cmp.get(u"reference", None)
1919         idx_cmp = cmp.get(u"compare", None)
1920         if idx_ref is None or idx_cmp is None:
1921             continue
1922         header[0].append(
1923             f"Diff({header[0][idx_ref - idx].split(u'~')[-1]} vs "
1924             f"{header[0][idx_cmp - idx].split(u'~')[-1]})"
1925         )
1926         header[1].append(u"")
1927         header[2].append(u"")
1928         header[3].append(u"")
1929         for tst_name, tst_data in tbl_dict.items():
1930             if not cmp_dict.get(tst_name, None):
1931                 cmp_dict[tst_name] = list()
1932             ref_data = tst_data.get(idx_ref, None)
1933             cmp_data = tst_data.get(idx_cmp, None)
1934             if ref_data is None or cmp_data is None:
1935                 cmp_dict[tst_name].append(float('nan'))
1936             else:
1937                 cmp_dict[tst_name].append(
1938                     relative_change(ref_data, cmp_data)
1939                 )
1940
1941     tbl_lst = list()
1942     for tst_name, tst_data in tbl_dict.items():
1943         itm_lst = [tst_data[u"name"], ]
1944         for idx in range(nr_cols):
1945             item = tst_data.get(-idx - 1, None)
1946             if item is None:
1947                 itm_lst.insert(1, None)
1948             else:
1949                 itm_lst.insert(1, round(item / 1e6, 1))
1950         itm_lst.extend(
1951             [
1952                 None if itm is None else round(itm, 1)
1953                 for itm in cmp_dict[tst_name]
1954             ]
1955         )
1956         tbl_lst.append(itm_lst)
1957
1958     tbl_lst.sort(key=lambda rel: rel[0], reverse=False)
1959     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1960
1961     # Generate csv table:
1962     csv_file = f"{table[u'output-file']}.csv"
1963     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1964         for hdr in header:
1965             file_handler.write(u",".join(hdr) + u"\n")
1966         for test in tbl_lst:
1967             file_handler.write(u",".join(
1968                 [
1969                     str(item).replace(u"None", u"-").replace(u"nan", u"-").
1970                     replace(u"null", u"-") for item in test
1971                 ]
1972             ) + u"\n")
1973
1974     txt_file = f"{table[u'output-file']}.txt"
1975     convert_csv_to_pretty_txt(csv_file, txt_file, delimiter=u",")
1976
1977     # Reorganize header in txt table
1978     txt_table = list()
1979     with open(txt_file, u"rt", encoding='utf-8') as file_handler:
1980         for line in file_handler:
1981             txt_table.append(line)
1982     try:
1983         txt_table.insert(5, txt_table.pop(2))
1984         with open(txt_file, u"wt", encoding='utf-8') as file_handler:
1985             file_handler.writelines(txt_table)
1986     except IndexError:
1987         pass
1988
1989     # Generate html table:
1990     hdr_html = [
1991         u"<br>".join(row) for row in zip(*header)
1992     ]
1993     _tpc_generate_html_table(
1994         hdr_html,
1995         tbl_lst,
1996         table[u'output-file'],
1997         sort_data=True,
1998         title=table.get(u"title", u""),
1999         generate_rst=False
2000     )