resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from collections import OrderedDict
  23 from xml.etree import ElementTree as ET
  24 from datetime import datetime as dt
  25 from datetime import timedelta
  26 from copy import deepcopy
  27
  28 import plotly.graph_objects as go
  29 import plotly.offline as ploff
  30 import pandas as pd
  31
  32 from numpy import nan, isnan
  33 from yaml import load, FullLoader, YAMLError
  34
  35 from pal_utils import mean, stdev, classify_anomalies, \
  36     convert_csv_to_pretty_txt, relative_change_stdev, relative_change
  37
  38
  39 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
  40
  41
  42 def generate_tables(spec, data):
  43     """Generate all tables specified in the specification file.
  44
  45     :param spec: Specification read from the specification file.
  46     :param data: Data to process.
  47     :type spec: Specification
  48     :type data: InputData
  49     """
  50
  51     generator = {
  52         u"table_merged_details": table_merged_details,
  53         u"table_soak_vs_ndr": table_soak_vs_ndr,
  54         u"table_perf_trending_dash": table_perf_trending_dash,
  55         u"table_perf_trending_dash_html": table_perf_trending_dash_html,
  56         u"table_last_failed_tests": table_last_failed_tests,
  57         u"table_failed_tests": table_failed_tests,
  58         u"table_failed_tests_html": table_failed_tests_html,
  59         u"table_oper_data_html": table_oper_data_html,
  60         u"table_comparison": table_comparison,
  61         u"table_weekly_comparison": table_weekly_comparison
  62     }
  63
  64     logging.info(u"Generating the tables ...")
  65     for table in spec.tables:
  66         try:
  67             if table[u"algorithm"] == u"table_weekly_comparison":
  68                 table[u"testbeds"] = spec.environment.get(u"testbeds", None)
  69             generator[table[u"algorithm"]](table, data)
  70         except NameError as err:
  71             logging.error(
  72                 f"Probably algorithm {table[u'algorithm']} is not defined: "
  73                 f"{repr(err)}"
  74             )
  75     logging.info(u"Done.")
  76
  77
  78 def table_oper_data_html(table, input_data):
  79     """Generate the table(s) with algorithm: html_table_oper_data
  80     specified in the specification file.
  81
  82     :param table: Table to generate.
  83     :param input_data: Data to process.
  84     :type table: pandas.Series
  85     :type input_data: InputData
  86     """
  87
  88     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
  89     # Transform the data
  90     logging.info(
  91         f"    Creating the data set for the {table.get(u'type', u'')} "
  92         f"{table.get(u'title', u'')}."
  93     )
  94     data = input_data.filter_data(
  95         table,
  96         params=[u"name", u"parent", u"show-run", u"type"],
  97         continue_on_error=True
  98     )
  99     if data.empty:
 100         return
 101     data = input_data.merge_data(data)
 102
 103     sort_tests = table.get(u"sort", None)
 104     if sort_tests:
 105         args = dict(
 106             inplace=True,
 107             ascending=(sort_tests == u"ascending")
 108         )
 109         data.sort_index(**args)
 110
 111     suites = input_data.filter_data(
 112         table,
 113         continue_on_error=True,
 114         data_set=u"suites"
 115     )
 116     if suites.empty:
 117         return
 118     suites = input_data.merge_data(suites)
 119
 120     def _generate_html_table(tst_data):
 121         """Generate an HTML table with operational data for the given test.
 122
 123         :param tst_data: Test data to be used to generate the table.
 124         :type tst_data: pandas.Series
 125         :returns: HTML table with operational data.
 126         :rtype: str
 127         """
 128
 129         colors = {
 130             u"header": u"#7eade7",
 131             u"empty": u"#ffffff",
 132             u"body": (u"#e9f1fb", u"#d4e4f7")
 133         }
 134
 135         tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
 136
 137         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
 138         thead = ET.SubElement(
 139             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 140         )
 141         thead.text = tst_data[u"name"]
 142
 143         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 144         thead = ET.SubElement(
 145             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 146         )
 147         thead.text = u"\t"
 148
 149         if tst_data.get(u"show-run", u"No Data") == u"No Data":
 150             trow = ET.SubElement(
 151                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 152             )
 153             tcol = ET.SubElement(
 154                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 155             )
 156             tcol.text = u"No Data"
 157
 158             trow = ET.SubElement(
 159                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 160             )
 161             thead = ET.SubElement(
 162                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 163             )
 164             font = ET.SubElement(
 165                 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 166             )
 167             font.text = u"."
 168             return str(ET.tostring(tbl, encoding=u"unicode"))
 169
 170         tbl_hdr = (
 171             u"Name",
 172             u"Nr of Vectors",
 173             u"Nr of Packets",
 174             u"Suspends",
 175             u"Cycles per Packet",
 176             u"Average Vector Size"
 177         )
 178
 179         for dut_data in tst_data[u"show-run"].values():
 180             trow = ET.SubElement(
 181                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 182             )
 183             tcol = ET.SubElement(
 184                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 185             )
 186             if dut_data.get(u"threads", None) is None:
 187                 tcol.text = u"No Data"
 188                 continue
 189
 190             bold = ET.SubElement(tcol, u"b")
 191             bold.text = (
 192                 f"Host IP: {dut_data.get(u'host', '')}, "
 193                 f"Socket: {dut_data.get(u'socket', '')}"
 194             )
 195             trow = ET.SubElement(
 196                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 197             )
 198             thead = ET.SubElement(
 199                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 200             )
 201             thead.text = u"\t"
 202
 203             for thread_nr, thread in dut_data[u"threads"].items():
 204                 trow = ET.SubElement(
 205                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 206                 )
 207                 tcol = ET.SubElement(
 208                     trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 209                 )
 210                 bold = ET.SubElement(tcol, u"b")
 211                 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
 212                 trow = ET.SubElement(
 213                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 214                 )
 215                 for idx, col in enumerate(tbl_hdr):
 216                     tcol = ET.SubElement(
 217                         trow, u"td",
 218                         attrib=dict(align=u"right" if idx else u"left")
 219                     )
 220                     font = ET.SubElement(
 221                         tcol, u"font", attrib=dict(size=u"2")
 222                     )
 223                     bold = ET.SubElement(font, u"b")
 224                     bold.text = col
 225                 for row_nr, row in enumerate(thread):
 226                     trow = ET.SubElement(
 227                         tbl, u"tr",
 228                         attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
 229                     )
 230                     for idx, col in enumerate(row):
 231                         tcol = ET.SubElement(
 232                             trow, u"td",
 233                             attrib=dict(align=u"right" if idx else u"left")
 234                         )
 235                         font = ET.SubElement(
 236                             tcol, u"font", attrib=dict(size=u"2")
 237                         )
 238                         if isinstance(col, float):
 239                             font.text = f"{col:.2f}"
 240                         else:
 241                             font.text = str(col)
 242                 trow = ET.SubElement(
 243                     tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 244                 )
 245                 thead = ET.SubElement(
 246                     trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 247                 )
 248                 thead.text = u"\t"
 249
 250         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 251         thead = ET.SubElement(
 252             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 253         )
 254         font = ET.SubElement(
 255             thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 256         )
 257         font.text = u"."
 258
 259         return str(ET.tostring(tbl, encoding=u"unicode"))
 260
 261     for suite in suites.values:
 262         html_table = str()
 263         for test_data in data.values:
 264             if test_data[u"parent"] not in suite[u"name"]:
 265                 continue
 266             html_table += _generate_html_table(test_data)
 267         if not html_table:
 268             continue
 269         try:
 270             file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
 271             with open(f"{file_name}", u'w') as html_file:
 272                 logging.info(f"    Writing file: {file_name}")
 273                 html_file.write(u".. raw:: html\n\n\t")
 274                 html_file.write(html_table)
 275                 html_file.write(u"\n\t<p><br><br></p>\n")
 276         except KeyError:
 277             logging.warning(u"The output file is not defined.")
 278             return
 279     logging.info(u"  Done.")
 280
 281
 282 def table_merged_details(table, input_data):
 283     """Generate the table(s) with algorithm: table_merged_details
 284     specified in the specification file.
 285
 286     :param table: Table to generate.
 287     :param input_data: Data to process.
 288     :type table: pandas.Series
 289     :type input_data: InputData
 290     """
 291
 292     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 293
 294     # Transform the data
 295     logging.info(
 296         f"    Creating the data set for the {table.get(u'type', u'')} "
 297         f"{table.get(u'title', u'')}."
 298     )
 299     data = input_data.filter_data(table, continue_on_error=True)
 300     data = input_data.merge_data(data)
 301
 302     sort_tests = table.get(u"sort", None)
 303     if sort_tests:
 304         args = dict(
 305             inplace=True,
 306             ascending=(sort_tests == u"ascending")
 307         )
 308         data.sort_index(**args)
 309
 310     suites = input_data.filter_data(
 311         table, continue_on_error=True, data_set=u"suites")
 312     suites = input_data.merge_data(suites)
 313
 314     # Prepare the header of the tables
 315     header = list()
 316     for column in table[u"columns"]:
 317         header.append(
 318             u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
 319         )
 320
 321     for suite in suites.values:
 322         # Generate data
 323         suite_name = suite[u"name"]
 324         table_lst = list()
 325         for test in data.keys():
 326             if data[test][u"parent"] not in suite_name:
 327                 continue
 328             row_lst = list()
 329             for column in table[u"columns"]:
 330                 try:
 331                     col_data = str(data[test][column[
 332                         u"data"].split(u" ")[1]]).replace(u'"', u'""')
 333                     # Do not include tests with "Test Failed" in test message
 334                     if u"Test Failed" in col_data:
 335                         continue
 336                     col_data = col_data.replace(
 337                         u"No Data", u"Not Captured     "
 338                     )
 339                     if column[u"data"].split(u" ")[1] in (u"name", ):
 340                         if len(col_data) > 30:
 341                             col_data_lst = col_data.split(u"-")
 342                             half = int(len(col_data_lst) / 2)
 343                             col_data = f"{u'-'.join(col_data_lst[:half])}" \
 344                                        f"- |br| " \
 345                                        f"{u'-'.join(col_data_lst[half:])}"
 346                         col_data = f" |prein| {col_data} |preout| "
 347                     elif column[u"data"].split(u" ")[1] in (u"msg", ):
 348                         # Temporary solution: remove NDR results from message:
 349                         if bool(table.get(u'remove-ndr', False)):
 350                             try:
 351                                 col_data = col_data.split(u" |br| ", 1)[1]
 352                             except IndexError:
 353                                 pass
 354                         col_data = f" |prein| {col_data} |preout| "
 355                     elif column[u"data"].split(u" ")[1] in \
 356                             (u"conf-history", u"show-run"):
 357                         col_data = col_data.replace(u" |br| ", u"", 1)
 358                         col_data = f" |prein| {col_data[:-5]} |preout| "
 359                     row_lst.append(f'"{col_data}"')
 360                 except KeyError:
 361                     row_lst.append(u'"Not captured"')
 362             if len(row_lst) == len(table[u"columns"]):
 363                 table_lst.append(row_lst)
 364
 365         # Write the data to file
 366         if table_lst:
 367             separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
 368             file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
 369             logging.info(f"      Writing file: {file_name}")
 370             with open(file_name, u"wt") as file_handler:
 371                 file_handler.write(u",".join(header) + u"\n")
 372                 for item in table_lst:
 373                     file_handler.write(u",".join(item) + u"\n")
 374
 375     logging.info(u"  Done.")
 376
 377
 378 def _tpc_modify_test_name(test_name, ignore_nic=False):
 379     """Modify a test name by replacing its parts.
 380
 381     :param test_name: Test name to be modified.
 382     :param ignore_nic: If True, NIC is removed from TC name.
 383     :type test_name: str
 384     :type ignore_nic: bool
 385     :returns: Modified test name.
 386     :rtype: str
 387     """
 388     test_name_mod = test_name.\
 389         replace(u"-ndrpdrdisc", u""). \
 390         replace(u"-ndrpdr", u"").\
 391         replace(u"-pdrdisc", u""). \
 392         replace(u"-ndrdisc", u"").\
 393         replace(u"-pdr", u""). \
 394         replace(u"-ndr", u""). \
 395         replace(u"1t1c", u"1c").\
 396         replace(u"2t1c", u"1c"). \
 397         replace(u"2t2c", u"2c").\
 398         replace(u"4t2c", u"2c"). \
 399         replace(u"4t4c", u"4c").\
 400         replace(u"8t4c", u"4c")
 401
 402     if ignore_nic:
 403         return re.sub(REGEX_NIC, u"", test_name_mod)
 404     return test_name_mod
 405
 406
 407 def _tpc_modify_displayed_test_name(test_name):
 408     """Modify a test name which is displayed in a table by replacing its parts.
 409
 410     :param test_name: Test name to be modified.
 411     :type test_name: str
 412     :returns: Modified test name.
 413     :rtype: str
 414     """
 415     return test_name.\
 416         replace(u"1t1c", u"1c").\
 417         replace(u"2t1c", u"1c"). \
 418         replace(u"2t2c", u"2c").\
 419         replace(u"4t2c", u"2c"). \
 420         replace(u"4t4c", u"4c").\
 421         replace(u"8t4c", u"4c")
 422
 423
 424 def _tpc_insert_data(target, src, include_tests):
 425     """Insert src data to the target structure.
 426
 427     :param target: Target structure where the data is placed.
 428     :param src: Source data to be placed into the target stucture.
 429     :param include_tests: Which results will be included (MRR, NDR, PDR).
 430     :type target: list
 431     :type src: dict
 432     :type include_tests: str
 433     """
 434     try:
 435         if include_tests == u"MRR":
 436             target[u"mean"] = src[u"result"][u"receive-rate"]
 437             target[u"stdev"] = src[u"result"][u"receive-stdev"]
 438         elif include_tests == u"PDR":
 439             target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
 440         elif include_tests == u"NDR":
 441             target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
 442     except (KeyError, TypeError):
 443         pass
 444
 445
 446 def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
 447                              footnote=u"", sort_data=True, title=u"",
 448                              generate_rst=True):
 449     """Generate html table from input data with simple sorting possibility.
 450
 451     :param header: Table header.
 452     :param data: Input data to be included in the table. It is a list of lists.
 453         Inner lists are rows in the table. All inner lists must be of the same
 454         length. The length of these lists must be the same as the length of the
 455         header.
 456     :param out_file_name: The name (relative or full path) where the
 457         generated html table is written.
 458     :param legend: The legend to display below the table.
 459     :param footnote: The footnote to display below the table (and legend).
 460     :param sort_data: If True the data sorting is enabled.
 461     :param title: The table (and file) title.
 462     :param generate_rst: If True, wrapping rst file is generated.
 463     :type header: list
 464     :type data: list of lists
 465     :type out_file_name: str
 466     :type legend: str
 467     :type footnote: str
 468     :type sort_data: bool
 469     :type title: str
 470     :type generate_rst: bool
 471     """
 472
 473     try:
 474         idx = header.index(u"Test Case")
 475     except ValueError:
 476         idx = 0
 477     params = {
 478         u"align-hdr": (
 479             [u"left", u"right"],
 480             [u"left", u"left", u"right"],
 481             [u"left", u"left", u"left", u"right"]
 482         ),
 483         u"align-itm": (
 484             [u"left", u"right"],
 485             [u"left", u"left", u"right"],
 486             [u"left", u"left", u"left", u"right"]
 487         ),
 488         u"width": ([15, 9], [4, 24, 10], [4, 4, 32, 10])
 489     }
 490
 491     df_data = pd.DataFrame(data, columns=header)
 492
 493     if sort_data:
 494         df_sorted = [df_data.sort_values(
 495             by=[key, header[idx]], ascending=[True, True]
 496             if key != header[idx] else [False, True]) for key in header]
 497         df_sorted_rev = [df_data.sort_values(
 498             by=[key, header[idx]], ascending=[False, True]
 499             if key != header[idx] else [True, True]) for key in header]
 500         df_sorted.extend(df_sorted_rev)
 501     else:
 502         df_sorted = df_data
 503
 504     fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
 505                    for idx in range(len(df_data))]]
 506     table_header = dict(
 507         values=[f"<b>{item.replace(u',', u',<br>')}</b>" for item in header],
 508         fill_color=u"#7eade7",
 509         align=params[u"align-hdr"][idx],
 510         font=dict(
 511             family=u"Courier New",
 512             size=12
 513         )
 514     )
 515
 516     fig = go.Figure()
 517
 518     if sort_data:
 519         for table in df_sorted:
 520             columns = [table.get(col) for col in header]
 521             fig.add_trace(
 522                 go.Table(
 523                     columnwidth=params[u"width"][idx],
 524                     header=table_header,
 525                     cells=dict(
 526                         values=columns,
 527                         fill_color=fill_color,
 528                         align=params[u"align-itm"][idx],
 529                         font=dict(
 530                             family=u"Courier New",
 531                             size=12
 532                         )
 533                     )
 534                 )
 535             )
 536
 537         buttons = list()
 538         menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
 539         menu_items.extend([f"<b>{itm}</b> (descending)" for itm in header])
 540         for idx, hdr in enumerate(menu_items):
 541             visible = [False, ] * len(menu_items)
 542             visible[idx] = True
 543             buttons.append(
 544                 dict(
 545                     label=hdr.replace(u" [Mpps]", u""),
 546                     method=u"update",
 547                     args=[{u"visible": visible}],
 548                 )
 549             )
 550
 551         fig.update_layout(
 552             updatemenus=[
 553                 go.layout.Updatemenu(
 554                     type=u"dropdown",
 555                     direction=u"down",
 556                     x=0.0,
 557                     xanchor=u"left",
 558                     y=1.002,
 559                     yanchor=u"bottom",
 560                     active=len(menu_items) - 1,
 561                     buttons=list(buttons)
 562                 )
 563             ],
 564         )
 565     else:
 566         fig.add_trace(
 567             go.Table(
 568                 columnwidth=params[u"width"][idx],
 569                 header=table_header,
 570                 cells=dict(
 571                     values=[df_sorted.get(col) for col in header],
 572                     fill_color=fill_color,
 573                     align=params[u"align-itm"][idx],
 574                     font=dict(
 575                         family=u"Courier New",
 576                         size=12
 577                     )
 578                 )
 579             )
 580         )
 581
 582     ploff.plot(
 583         fig,
 584         show_link=False,
 585         auto_open=False,
 586         filename=f"{out_file_name}_in.html"
 587     )
 588
 589     if not generate_rst:
 590         return
 591
 592     file_name = out_file_name.split(u"/")[-1]
 593     if u"vpp" in out_file_name:
 594         path = u"_tmp/src/vpp_performance_tests/comparisons/"
 595     else:
 596         path = u"_tmp/src/dpdk_performance_tests/comparisons/"
 597     logging.info(f"    Writing the HTML file to {path}{file_name}.rst")
 598     with open(f"{path}{file_name}.rst", u"wt") as rst_file:
 599         rst_file.write(
 600             u"\n"
 601             u".. |br| raw:: html\n\n    <br />\n\n\n"
 602             u".. |prein| raw:: html\n\n    <pre>\n\n\n"
 603             u".. |preout| raw:: html\n\n    </pre>\n\n"
 604         )
 605         if title:
 606             rst_file.write(f"{title}\n")
 607             rst_file.write(f"{u'`' * len(title)}\n\n")
 608         rst_file.write(
 609             u".. raw:: html\n\n"
 610             f'    <iframe frameborder="0" scrolling="no" '
 611             f'width="1600" height="1200" '
 612             f'src="../..{out_file_name.replace(u"_build", u"")}_in.html">'
 613             f'</iframe>\n\n'
 614         )
 615
 616         if legend:
 617             try:
 618                 itm_lst = legend[1:-2].split(u"\n")
 619                 rst_file.write(
 620                     f"{itm_lst[0]}\n\n- " + u'\n- '.join(itm_lst[1:]) + u"\n\n"
 621                 )
 622             except IndexError as err:
 623                 logging.error(f"Legend cannot be written to html file\n{err}")
 624         if footnote:
 625             try:
 626                 itm_lst = footnote[1:].split(u"\n")
 627                 rst_file.write(
 628                     f"{itm_lst[0]}\n\n- " + u'\n- '.join(itm_lst[1:]) + u"\n\n"
 629                 )
 630             except IndexError as err:
 631                 logging.error(f"Footnote cannot be written to html file\n{err}")
 632
 633
 634 def table_soak_vs_ndr(table, input_data):
 635     """Generate the table(s) with algorithm: table_soak_vs_ndr
 636     specified in the specification file.
 637
 638     :param table: Table to generate.
 639     :param input_data: Data to process.
 640     :type table: pandas.Series
 641     :type input_data: InputData
 642     """
 643
 644     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 645
 646     # Transform the data
 647     logging.info(
 648         f"    Creating the data set for the {table.get(u'type', u'')} "
 649         f"{table.get(u'title', u'')}."
 650     )
 651     data = input_data.filter_data(table, continue_on_error=True)
 652
 653     # Prepare the header of the table
 654     try:
 655         header = [
 656             u"Test Case",
 657             f"Avg({table[u'reference'][u'title']})",
 658             f"Stdev({table[u'reference'][u'title']})",
 659             f"Avg({table[u'compare'][u'title']})",
 660             f"Stdev{table[u'compare'][u'title']})",
 661             u"Diff",
 662             u"Stdev(Diff)"
 663         ]
 664         header_str = u";".join(header) + u"\n"
 665         legend = (
 666             u"\nLegend:\n"
 667             f"Avg({table[u'reference'][u'title']}): "
 668             f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
 669             f"from a series of runs of the listed tests.\n"
 670             f"Stdev({table[u'reference'][u'title']}): "
 671             f"Standard deviation value of {table[u'reference'][u'title']} "
 672             f"[Mpps] computed from a series of runs of the listed tests.\n"
 673             f"Avg({table[u'compare'][u'title']}): "
 674             f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
 675             f"a series of runs of the listed tests.\n"
 676             f"Stdev({table[u'compare'][u'title']}): "
 677             f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
 678             f"computed from a series of runs of the listed tests.\n"
 679             f"Diff({table[u'reference'][u'title']},"
 680             f"{table[u'compare'][u'title']}): "
 681             f"Percentage change calculated for mean values.\n"
 682             u"Stdev(Diff): "
 683             u"Standard deviation of percentage change calculated for mean "
 684             u"values."
 685         )
 686     except (AttributeError, KeyError) as err:
 687         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 688         return
 689
 690     # Create a list of available SOAK test results:
 691     tbl_dict = dict()
 692     for job, builds in table[u"compare"][u"data"].items():
 693         for build in builds:
 694             for tst_name, tst_data in data[job][str(build)].items():
 695                 if tst_data[u"type"] == u"SOAK":
 696                     tst_name_mod = tst_name.replace(u"-soak", u"")
 697                     if tbl_dict.get(tst_name_mod, None) is None:
 698                         groups = re.search(REGEX_NIC, tst_data[u"parent"])
 699                         nic = groups.group(0) if groups else u""
 700                         name = (
 701                             f"{nic}-"
 702                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 703                         )
 704                         tbl_dict[tst_name_mod] = {
 705                             u"name": name,
 706                             u"ref-data": list(),
 707                             u"cmp-data": list()
 708                         }
 709                     try:
 710                         tbl_dict[tst_name_mod][u"cmp-data"].append(
 711                             tst_data[u"throughput"][u"LOWER"])
 712                     except (KeyError, TypeError):
 713                         pass
 714     tests_lst = tbl_dict.keys()
 715
 716     # Add corresponding NDR test results:
 717     for job, builds in table[u"reference"][u"data"].items():
 718         for build in builds:
 719             for tst_name, tst_data in data[job][str(build)].items():
 720                 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
 721                     replace(u"-mrr", u"")
 722                 if tst_name_mod not in tests_lst:
 723                     continue
 724                 try:
 725                     if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
 726                         continue
 727                     if table[u"include-tests"] == u"MRR":
 728                         result = (tst_data[u"result"][u"receive-rate"],
 729                                   tst_data[u"result"][u"receive-stdev"])
 730                     elif table[u"include-tests"] == u"PDR":
 731                         result = \
 732                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
 733                     elif table[u"include-tests"] == u"NDR":
 734                         result = \
 735                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
 736                     else:
 737                         result = None
 738                     if result is not None:
 739                         tbl_dict[tst_name_mod][u"ref-data"].append(
 740                             result)
 741                 except (KeyError, TypeError):
 742                     continue
 743
 744     tbl_lst = list()
 745     for tst_name in tbl_dict:
 746         item = [tbl_dict[tst_name][u"name"], ]
 747         data_r = tbl_dict[tst_name][u"ref-data"]
 748         if data_r:
 749             if table[u"include-tests"] == u"MRR":
 750                 data_r_mean = data_r[0][0]
 751                 data_r_stdev = data_r[0][1]
 752             else:
 753                 data_r_mean = mean(data_r)
 754                 data_r_stdev = stdev(data_r)
 755             item.append(round(data_r_mean / 1e6, 1))
 756             item.append(round(data_r_stdev / 1e6, 1))
 757         else:
 758             data_r_mean = None
 759             data_r_stdev = None
 760             item.extend([None, None])
 761         data_c = tbl_dict[tst_name][u"cmp-data"]
 762         if data_c:
 763             if table[u"include-tests"] == u"MRR":
 764                 data_c_mean = data_c[0][0]
 765                 data_c_stdev = data_c[0][1]
 766             else:
 767                 data_c_mean = mean(data_c)
 768                 data_c_stdev = stdev(data_c)
 769             item.append(round(data_c_mean / 1e6, 1))
 770             item.append(round(data_c_stdev / 1e6, 1))
 771         else:
 772             data_c_mean = None
 773             data_c_stdev = None
 774             item.extend([None, None])
 775         if data_r_mean is not None and data_c_mean is not None:
 776             delta, d_stdev = relative_change_stdev(
 777                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
 778             try:
 779                 item.append(round(delta))
 780             except ValueError:
 781                 item.append(delta)
 782             try:
 783                 item.append(round(d_stdev))
 784             except ValueError:
 785                 item.append(d_stdev)
 786             tbl_lst.append(item)
 787
 788     # Sort the table according to the relative change
 789     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 790
 791     # Generate csv tables:
 792     csv_file_name = f"{table[u'output-file']}.csv"
 793     with open(csv_file_name, u"wt") as file_handler:
 794         file_handler.write(header_str)
 795         for test in tbl_lst:
 796             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 797
 798     convert_csv_to_pretty_txt(
 799         csv_file_name, f"{table[u'output-file']}.txt", delimiter=u";"
 800     )
 801     with open(f"{table[u'output-file']}.txt", u'a') as file_handler:
 802         file_handler.write(legend)
 803
 804     # Generate html table:
 805     _tpc_generate_html_table(
 806         header,
 807         tbl_lst,
 808         table[u'output-file'],
 809         legend=legend,
 810         title=table.get(u"title", u"")
 811     )
 812
 813
 814 def table_perf_trending_dash(table, input_data):
 815     """Generate the table(s) with algorithm:
 816     table_perf_trending_dash
 817     specified in the specification file.
 818
 819     :param table: Table to generate.
 820     :param input_data: Data to process.
 821     :type table: pandas.Series
 822     :type input_data: InputData
 823     """
 824
 825     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 826
 827     # Transform the data
 828     logging.info(
 829         f"    Creating the data set for the {table.get(u'type', u'')} "
 830         f"{table.get(u'title', u'')}."
 831     )
 832     data = input_data.filter_data(table, continue_on_error=True)
 833
 834     # Prepare the header of the tables
 835     header = [
 836         u"Test Case",
 837         u"Trend [Mpps]",
 838         u"Short-Term Change [%]",
 839         u"Long-Term Change [%]",
 840         u"Regressions [#]",
 841         u"Progressions [#]"
 842     ]
 843     header_str = u",".join(header) + u"\n"
 844
 845     incl_tests = table.get(u"include-tests", u"MRR")
 846
 847     # Prepare data to the table:
 848     tbl_dict = dict()
 849     for job, builds in table[u"data"].items():
 850         for build in builds:
 851             for tst_name, tst_data in data[job][str(build)].items():
 852                 if tst_name.lower() in table.get(u"ignore-list", list()):
 853                     continue
 854                 if tbl_dict.get(tst_name, None) is None:
 855                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
 856                     if not groups:
 857                         continue
 858                     nic = groups.group(0)
 859                     tbl_dict[tst_name] = {
 860                         u"name": f"{nic}-{tst_data[u'name']}",
 861                         u"data": OrderedDict()
 862                     }
 863                 try:
 864                     if incl_tests == u"MRR":
 865                         tbl_dict[tst_name][u"data"][str(build)] = \
 866                             tst_data[u"result"][u"receive-rate"]
 867                     elif incl_tests == u"NDR":
 868                         tbl_dict[tst_name][u"data"][str(build)] = \
 869                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
 870                     elif incl_tests == u"PDR":
 871                         tbl_dict[tst_name][u"data"][str(build)] = \
 872                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
 873                 except (TypeError, KeyError):
 874                     pass  # No data in output.xml for this test
 875
 876     tbl_lst = list()
 877     for tst_name in tbl_dict:
 878         data_t = tbl_dict[tst_name][u"data"]
 879         if len(data_t) < 2:
 880             continue
 881
 882         classification_lst, avgs, _ = classify_anomalies(data_t)
 883
 884         win_size = min(len(data_t), table[u"window"])
 885         long_win_size = min(len(data_t), table[u"long-trend-window"])
 886
 887         try:
 888             max_long_avg = max(
 889                 [x for x in avgs[-long_win_size:-win_size]
 890                  if not isnan(x)])
 891         except ValueError:
 892             max_long_avg = nan
 893         last_avg = avgs[-1]
 894         avg_week_ago = avgs[max(-win_size, -len(avgs))]
 895
 896         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
 897             rel_change_last = nan
 898         else:
 899             rel_change_last = round(
 900                 ((last_avg - avg_week_ago) / avg_week_ago) * 1e2, 2)
 901
 902         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
 903             rel_change_long = nan
 904         else:
 905             rel_change_long = round(
 906                 ((last_avg - max_long_avg) / max_long_avg) * 1e2, 2)
 907
 908         if classification_lst:
 909             if isnan(rel_change_last) and isnan(rel_change_long):
 910                 continue
 911             if isnan(last_avg) or isnan(rel_change_last) or \
 912                     isnan(rel_change_long):
 913                 continue
 914             tbl_lst.append(
 915                 [tbl_dict[tst_name][u"name"],
 916                  round(last_avg / 1e6, 2),
 917                  rel_change_last,
 918                  rel_change_long,
 919                  classification_lst[-win_size+1:].count(u"regression"),
 920                  classification_lst[-win_size+1:].count(u"progression")])
 921
 922     tbl_lst.sort(key=lambda rel: rel[0])
 923
 924     tbl_sorted = list()
 925     for nrr in range(table[u"window"], -1, -1):
 926         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
 927         for nrp in range(table[u"window"], -1, -1):
 928             tbl_out = [item for item in tbl_reg if item[5] == nrp]
 929             tbl_out.sort(key=lambda rel: rel[2])
 930             tbl_sorted.extend(tbl_out)
 931
 932     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
 933
 934     logging.info(f"    Writing file: {file_name}")
 935     with open(file_name, u"wt") as file_handler:
 936         file_handler.write(header_str)
 937         for test in tbl_sorted:
 938             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
 939
 940     logging.info(f"    Writing file: {table[u'output-file']}.txt")
 941     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
 942
 943
 944 def _generate_url(testbed, test_name):
 945     """Generate URL to a trending plot from the name of the test case.
 946
 947     :param testbed: The testbed used for testing.
 948     :param test_name: The name of the test case.
 949     :type testbed: str
 950     :type test_name: str
 951     :returns: The URL to the plot with the trending data for the given test
 952         case.
 953     :rtype str
 954     """
 955
 956     if u"x520" in test_name:
 957         nic = u"x520"
 958     elif u"x710" in test_name:
 959         nic = u"x710"
 960     elif u"xl710" in test_name:
 961         nic = u"xl710"
 962     elif u"xxv710" in test_name:
 963         nic = u"xxv710"
 964     elif u"vic1227" in test_name:
 965         nic = u"vic1227"
 966     elif u"vic1385" in test_name:
 967         nic = u"vic1385"
 968     elif u"x553" in test_name:
 969         nic = u"x553"
 970     elif u"cx556" in test_name or u"cx556a" in test_name:
 971         nic = u"cx556a"
 972     else:
 973         nic = u""
 974
 975     if u"64b" in test_name:
 976         frame_size = u"64b"
 977     elif u"78b" in test_name:
 978         frame_size = u"78b"
 979     elif u"imix" in test_name:
 980         frame_size = u"imix"
 981     elif u"9000b" in test_name:
 982         frame_size = u"9000b"
 983     elif u"1518b" in test_name:
 984         frame_size = u"1518b"
 985     elif u"114b" in test_name:
 986         frame_size = u"114b"
 987     else:
 988         frame_size = u""
 989
 990     if u"1t1c" in test_name or \
 991         (u"-1c-" in test_name and
 992          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 993         cores = u"1t1c"
 994     elif u"2t2c" in test_name or \
 995          (u"-2c-" in test_name and
 996           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 997         cores = u"2t2c"
 998     elif u"4t4c" in test_name or \
 999          (u"-4c-" in test_name and
1000           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
1001         cores = u"4t4c"
1002     elif u"2t1c" in test_name or \
1003          (u"-1c-" in test_name and
1004           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1005         cores = u"2t1c"
1006     elif u"4t2c" in test_name or \
1007          (u"-2c-" in test_name and
1008           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1009         cores = u"4t2c"
1010     elif u"8t4c" in test_name or \
1011          (u"-4c-" in test_name and
1012           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1013         cores = u"8t4c"
1014     else:
1015         cores = u""
1016
1017     if u"testpmd" in test_name:
1018         driver = u"testpmd"
1019     elif u"l3fwd" in test_name:
1020         driver = u"l3fwd"
1021     elif u"avf" in test_name:
1022         driver = u"avf"
1023     elif u"rdma" in test_name:
1024         driver = u"rdma"
1025     elif u"dnv" in testbed or u"tsh" in testbed:
1026         driver = u"ixgbe"
1027     else:
1028         driver = u"dpdk"
1029
1030     if u"acl" in test_name or \
1031             u"macip" in test_name or \
1032             u"nat" in test_name or \
1033             u"policer" in test_name or \
1034             u"cop" in test_name:
1035         bsf = u"features"
1036     elif u"scale" in test_name:
1037         bsf = u"scale"
1038     elif u"base" in test_name:
1039         bsf = u"base"
1040     else:
1041         bsf = u"base"
1042
1043     if u"114b" in test_name and u"vhost" in test_name:
1044         domain = u"vts"
1045     elif u"testpmd" in test_name or u"l3fwd" in test_name:
1046         domain = u"dpdk"
1047     elif u"memif" in test_name:
1048         domain = u"container_memif"
1049     elif u"srv6" in test_name:
1050         domain = u"srv6"
1051     elif u"vhost" in test_name:
1052         domain = u"vhost"
1053         if u"vppl2xc" in test_name:
1054             driver += u"-vpp"
1055         else:
1056             driver += u"-testpmd"
1057         if u"lbvpplacp" in test_name:
1058             bsf += u"-link-bonding"
1059     elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1060         domain = u"nf_service_density_vnfc"
1061     elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1062         domain = u"nf_service_density_cnfc"
1063     elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1064         domain = u"nf_service_density_cnfp"
1065     elif u"ipsec" in test_name:
1066         domain = u"ipsec"
1067         if u"sw" in test_name:
1068             bsf += u"-sw"
1069         elif u"hw" in test_name:
1070             bsf += u"-hw"
1071     elif u"ethip4vxlan" in test_name:
1072         domain = u"ip4_tunnels"
1073     elif u"ip4base" in test_name or u"ip4scale" in test_name:
1074         domain = u"ip4"
1075     elif u"ip6base" in test_name or u"ip6scale" in test_name:
1076         domain = u"ip6"
1077     elif u"l2xcbase" in test_name or \
1078             u"l2xcscale" in test_name or \
1079             u"l2bdbasemaclrn" in test_name or \
1080             u"l2bdscale" in test_name or \
1081             u"l2patch" in test_name:
1082         domain = u"l2"
1083     else:
1084         domain = u""
1085
1086     file_name = u"-".join((domain, testbed, nic)) + u".html#"
1087     anchor_name = u"-".join((frame_size, cores, bsf, driver))
1088
1089     return file_name + anchor_name
1090
1091
1092 def table_perf_trending_dash_html(table, input_data):
1093     """Generate the table(s) with algorithm:
1094     table_perf_trending_dash_html specified in the specification
1095     file.
1096
1097     :param table: Table to generate.
1098     :param input_data: Data to process.
1099     :type table: dict
1100     :type input_data: InputData
1101     """
1102
1103     _ = input_data
1104
1105     if not table.get(u"testbed", None):
1106         logging.error(
1107             f"The testbed is not defined for the table "
1108             f"{table.get(u'title', u'')}."
1109         )
1110         return
1111
1112     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1113
1114     try:
1115         with open(table[u"input-file"], u'rt') as csv_file:
1116             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1117     except KeyError:
1118         logging.warning(u"The input file is not defined.")
1119         return
1120     except csv.Error as err:
1121         logging.warning(
1122             f"Not possible to process the file {table[u'input-file']}.\n"
1123             f"{repr(err)}"
1124         )
1125         return
1126
1127     # Table:
1128     dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1129
1130     # Table header:
1131     trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1132     for idx, item in enumerate(csv_lst[0]):
1133         alignment = u"left" if idx == 0 else u"center"
1134         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1135         thead.text = item
1136
1137     # Rows:
1138     colors = {
1139         u"regression": (
1140             u"#ffcccc",
1141             u"#ff9999"
1142         ),
1143         u"progression": (
1144             u"#c6ecc6",
1145             u"#9fdf9f"
1146         ),
1147         u"normal": (
1148             u"#e9f1fb",
1149             u"#d4e4f7"
1150         )
1151     }
1152     for r_idx, row in enumerate(csv_lst[1:]):
1153         if int(row[4]):
1154             color = u"regression"
1155         elif int(row[5]):
1156             color = u"progression"
1157         else:
1158             color = u"normal"
1159         trow = ET.SubElement(
1160             dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
1161         )
1162
1163         # Columns:
1164         for c_idx, item in enumerate(row):
1165             tdata = ET.SubElement(
1166                 trow,
1167                 u"td",
1168                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1169             )
1170             # Name:
1171             if c_idx == 0 and table.get(u"add-links", True):
1172                 ref = ET.SubElement(
1173                     tdata,
1174                     u"a",
1175                     attrib=dict(
1176                         href=f"../trending/"
1177                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1178                     )
1179                 )
1180                 ref.text = item
1181             else:
1182                 tdata.text = item
1183     try:
1184         with open(table[u"output-file"], u'w') as html_file:
1185             logging.info(f"    Writing file: {table[u'output-file']}")
1186             html_file.write(u".. raw:: html\n\n\t")
1187             html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
1188             html_file.write(u"\n\t<p><br><br></p>\n")
1189     except KeyError:
1190         logging.warning(u"The output file is not defined.")
1191         return
1192
1193
1194 def table_last_failed_tests(table, input_data):
1195     """Generate the table(s) with algorithm: table_last_failed_tests
1196     specified in the specification file.
1197
1198     :param table: Table to generate.
1199     :param input_data: Data to process.
1200     :type table: pandas.Series
1201     :type input_data: InputData
1202     """
1203
1204     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1205
1206     # Transform the data
1207     logging.info(
1208         f"    Creating the data set for the {table.get(u'type', u'')} "
1209         f"{table.get(u'title', u'')}."
1210     )
1211
1212     data = input_data.filter_data(table, continue_on_error=True)
1213
1214     if data is None or data.empty:
1215         logging.warning(
1216             f"    No data for the {table.get(u'type', u'')} "
1217             f"{table.get(u'title', u'')}."
1218         )
1219         return
1220
1221     tbl_list = list()
1222     for job, builds in table[u"data"].items():
1223         for build in builds:
1224             build = str(build)
1225             try:
1226                 version = input_data.metadata(job, build).get(u"version", u"")
1227             except KeyError:
1228                 logging.error(f"Data for {job}: {build} is not present.")
1229                 return
1230             tbl_list.append(build)
1231             tbl_list.append(version)
1232             failed_tests = list()
1233             passed = 0
1234             failed = 0
1235             for tst_data in data[job][build].values:
1236                 if tst_data[u"status"] != u"FAIL":
1237                     passed += 1
1238                     continue
1239                 failed += 1
1240                 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1241                 if not groups:
1242                     continue
1243                 nic = groups.group(0)
1244                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
1245             tbl_list.append(str(passed))
1246             tbl_list.append(str(failed))
1247             tbl_list.extend(failed_tests)
1248
1249     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1250     logging.info(f"    Writing file: {file_name}")
1251     with open(file_name, u"wt") as file_handler:
1252         for test in tbl_list:
1253             file_handler.write(test + u'\n')
1254
1255
1256 def table_failed_tests(table, input_data):
1257     """Generate the table(s) with algorithm: table_failed_tests
1258     specified in the specification file.
1259
1260     :param table: Table to generate.
1261     :param input_data: Data to process.
1262     :type table: pandas.Series
1263     :type input_data: InputData
1264     """
1265
1266     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1267
1268     # Transform the data
1269     logging.info(
1270         f"    Creating the data set for the {table.get(u'type', u'')} "
1271         f"{table.get(u'title', u'')}."
1272     )
1273     data = input_data.filter_data(table, continue_on_error=True)
1274
1275     # Prepare the header of the tables
1276     header = [
1277         u"Test Case",
1278         u"Failures [#]",
1279         u"Last Failure [Time]",
1280         u"Last Failure [VPP-Build-Id]",
1281         u"Last Failure [CSIT-Job-Build-Id]"
1282     ]
1283
1284     # Generate the data for the table according to the model in the table
1285     # specification
1286
1287     now = dt.utcnow()
1288     timeperiod = timedelta(int(table.get(u"window", 7)))
1289
1290     tbl_dict = dict()
1291     for job, builds in table[u"data"].items():
1292         for build in builds:
1293             build = str(build)
1294             for tst_name, tst_data in data[job][build].items():
1295                 if tst_name.lower() in table.get(u"ignore-list", list()):
1296                     continue
1297                 if tbl_dict.get(tst_name, None) is None:
1298                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1299                     if not groups:
1300                         continue
1301                     nic = groups.group(0)
1302                     tbl_dict[tst_name] = {
1303                         u"name": f"{nic}-{tst_data[u'name']}",
1304                         u"data": OrderedDict()
1305                     }
1306                 try:
1307                     generated = input_data.metadata(job, build).\
1308                         get(u"generated", u"")
1309                     if not generated:
1310                         continue
1311                     then = dt.strptime(generated, u"%Y%m%d %H:%M")
1312                     if (now - then) <= timeperiod:
1313                         tbl_dict[tst_name][u"data"][build] = (
1314                             tst_data[u"status"],
1315                             generated,
1316                             input_data.metadata(job, build).get(u"version",
1317                                                                 u""),
1318                             build
1319                         )
1320                 except (TypeError, KeyError) as err:
1321                     logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
1322
1323     max_fails = 0
1324     tbl_lst = list()
1325     for tst_data in tbl_dict.values():
1326         fails_nr = 0
1327         fails_last_date = u""
1328         fails_last_vpp = u""
1329         fails_last_csit = u""
1330         for val in tst_data[u"data"].values():
1331             if val[0] == u"FAIL":
1332                 fails_nr += 1
1333                 fails_last_date = val[1]
1334                 fails_last_vpp = val[2]
1335                 fails_last_csit = val[3]
1336         if fails_nr:
1337             max_fails = fails_nr if fails_nr > max_fails else max_fails
1338             tbl_lst.append(
1339                 [
1340                     tst_data[u"name"],
1341                     fails_nr,
1342                     fails_last_date,
1343                     fails_last_vpp,
1344                     f"mrr-daily-build-{fails_last_csit}"
1345                 ]
1346             )
1347
1348     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1349     tbl_sorted = list()
1350     for nrf in range(max_fails, -1, -1):
1351         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1352         tbl_sorted.extend(tbl_fails)
1353
1354     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1355     logging.info(f"    Writing file: {file_name}")
1356     with open(file_name, u"wt") as file_handler:
1357         file_handler.write(u",".join(header) + u"\n")
1358         for test in tbl_sorted:
1359             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1360
1361     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1362     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1363
1364
1365 def table_failed_tests_html(table, input_data):
1366     """Generate the table(s) with algorithm: table_failed_tests_html
1367     specified in the specification file.
1368
1369     :param table: Table to generate.
1370     :param input_data: Data to process.
1371     :type table: pandas.Series
1372     :type input_data: InputData
1373     """
1374
1375     _ = input_data
1376
1377     if not table.get(u"testbed", None):
1378         logging.error(
1379             f"The testbed is not defined for the table "
1380             f"{table.get(u'title', u'')}."
1381         )
1382         return
1383
1384     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1385
1386     try:
1387         with open(table[u"input-file"], u'rt') as csv_file:
1388             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1389     except KeyError:
1390         logging.warning(u"The input file is not defined.")
1391         return
1392     except csv.Error as err:
1393         logging.warning(
1394             f"Not possible to process the file {table[u'input-file']}.\n"
1395             f"{repr(err)}"
1396         )
1397         return
1398
1399     # Table:
1400     failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1401
1402     # Table header:
1403     trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1404     for idx, item in enumerate(csv_lst[0]):
1405         alignment = u"left" if idx == 0 else u"center"
1406         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1407         thead.text = item
1408
1409     # Rows:
1410     colors = (u"#e9f1fb", u"#d4e4f7")
1411     for r_idx, row in enumerate(csv_lst[1:]):
1412         background = colors[r_idx % 2]
1413         trow = ET.SubElement(
1414             failed_tests, u"tr", attrib=dict(bgcolor=background)
1415         )
1416
1417         # Columns:
1418         for c_idx, item in enumerate(row):
1419             tdata = ET.SubElement(
1420                 trow,
1421                 u"td",
1422                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1423             )
1424             # Name:
1425             if c_idx == 0:
1426                 ref = ET.SubElement(
1427                     tdata,
1428                     u"a",
1429                     attrib=dict(
1430                         href=f"../trending/"
1431                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1432                     )
1433                 )
1434                 ref.text = item
1435             else:
1436                 tdata.text = item
1437     try:
1438         with open(table[u"output-file"], u'w') as html_file:
1439             logging.info(f"    Writing file: {table[u'output-file']}")
1440             html_file.write(u".. raw:: html\n\n\t")
1441             html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
1442             html_file.write(u"\n\t<p><br><br></p>\n")
1443     except KeyError:
1444         logging.warning(u"The output file is not defined.")
1445         return
1446
1447
1448 def table_comparison(table, input_data):
1449     """Generate the table(s) with algorithm: table_comparison
1450     specified in the specification file.
1451
1452     :param table: Table to generate.
1453     :param input_data: Data to process.
1454     :type table: pandas.Series
1455     :type input_data: InputData
1456     """
1457     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1458
1459     # Transform the data
1460     logging.info(
1461         f"    Creating the data set for the {table.get(u'type', u'')} "
1462         f"{table.get(u'title', u'')}."
1463     )
1464
1465     columns = table.get(u"columns", None)
1466     if not columns:
1467         logging.error(
1468             f"No columns specified for {table.get(u'title', u'')}. Skipping."
1469         )
1470         return
1471
1472     cols = list()
1473     for idx, col in enumerate(columns):
1474         if col.get(u"data-set", None) is None:
1475             logging.warning(f"No data for column {col.get(u'title', u'')}")
1476             continue
1477         tag = col.get(u"tag", None)
1478         data = input_data.filter_data(
1479             table,
1480             params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1481             data=col[u"data-set"],
1482             continue_on_error=True
1483         )
1484         col_data = {
1485             u"title": col.get(u"title", f"Column{idx}"),
1486             u"data": dict()
1487         }
1488         for builds in data.values:
1489             for build in builds:
1490                 for tst_name, tst_data in build.items():
1491                     if tag and tag not in tst_data[u"tags"]:
1492                         continue
1493                     tst_name_mod = \
1494                         _tpc_modify_test_name(tst_name, ignore_nic=True).\
1495                         replace(u"2n1l-", u"")
1496                     if col_data[u"data"].get(tst_name_mod, None) is None:
1497                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
1498                         if u"across testbeds" in table[u"title"].lower() or \
1499                                 u"across topologies" in table[u"title"].lower():
1500                             name = _tpc_modify_displayed_test_name(name)
1501                         col_data[u"data"][tst_name_mod] = {
1502                             u"name": name,
1503                             u"replace": True,
1504                             u"data": list(),
1505                             u"mean": None,
1506                             u"stdev": None
1507                         }
1508                     _tpc_insert_data(
1509                         target=col_data[u"data"][tst_name_mod],
1510                         src=tst_data,
1511                         include_tests=table[u"include-tests"]
1512                     )
1513
1514         replacement = col.get(u"data-replacement", None)
1515         if replacement:
1516             rpl_data = input_data.filter_data(
1517                 table,
1518                 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1519                 data=replacement,
1520                 continue_on_error=True
1521             )
1522             for builds in rpl_data.values:
1523                 for build in builds:
1524                     for tst_name, tst_data in build.items():
1525                         if tag and tag not in tst_data[u"tags"]:
1526                             continue
1527                         tst_name_mod = \
1528                             _tpc_modify_test_name(tst_name, ignore_nic=True).\
1529                             replace(u"2n1l-", u"")
1530                         if col_data[u"data"].get(tst_name_mod, None) is None:
1531                             name = tst_data[u'name'].rsplit(u'-', 1)[0]
1532                             if u"across testbeds" in table[u"title"].lower() \
1533                                     or u"across topologies" in \
1534                                     table[u"title"].lower():
1535                                 name = _tpc_modify_displayed_test_name(name)
1536                             col_data[u"data"][tst_name_mod] = {
1537                                 u"name": name,
1538                                 u"replace": False,
1539                                 u"data": list(),
1540                                 u"mean": None,
1541                                 u"stdev": None
1542                             }
1543                         if col_data[u"data"][tst_name_mod][u"replace"]:
1544                             col_data[u"data"][tst_name_mod][u"replace"] = False
1545                             col_data[u"data"][tst_name_mod][u"data"] = list()
1546                         _tpc_insert_data(
1547                             target=col_data[u"data"][tst_name_mod],
1548                             src=tst_data,
1549                             include_tests=table[u"include-tests"]
1550                         )
1551
1552         if table[u"include-tests"] in (u"NDR", u"PDR"):
1553             for tst_name, tst_data in col_data[u"data"].items():
1554                 if tst_data[u"data"]:
1555                     tst_data[u"mean"] = mean(tst_data[u"data"])
1556                     tst_data[u"stdev"] = stdev(tst_data[u"data"])
1557
1558         cols.append(col_data)
1559
1560     tbl_dict = dict()
1561     for col in cols:
1562         for tst_name, tst_data in col[u"data"].items():
1563             if tbl_dict.get(tst_name, None) is None:
1564                 tbl_dict[tst_name] = {
1565                     "name": tst_data[u"name"]
1566                 }
1567             tbl_dict[tst_name][col[u"title"]] = {
1568                 u"mean": tst_data[u"mean"],
1569                 u"stdev": tst_data[u"stdev"]
1570             }
1571
1572     if not tbl_dict:
1573         logging.warning(f"No data for table {table.get(u'title', u'')}!")
1574         return
1575
1576     tbl_lst = list()
1577     for tst_data in tbl_dict.values():
1578         row = [tst_data[u"name"], ]
1579         for col in cols:
1580             row.append(tst_data.get(col[u"title"], None))
1581         tbl_lst.append(row)
1582
1583     comparisons = table.get(u"comparisons", None)
1584     if comparisons and isinstance(comparisons, list):
1585         for idx, comp in enumerate(comparisons):
1586             try:
1587                 col_ref = int(comp[u"reference"])
1588                 col_cmp = int(comp[u"compare"])
1589             except KeyError:
1590                 logging.warning(u"Comparison: No references defined! Skipping.")
1591                 comparisons.pop(idx)
1592                 continue
1593             if not (0 < col_ref <= len(cols) and
1594                     0 < col_cmp <= len(cols)) or \
1595                     col_ref == col_cmp:
1596                 logging.warning(f"Wrong values of reference={col_ref} "
1597                                 f"and/or compare={col_cmp}. Skipping.")
1598                 comparisons.pop(idx)
1599                 continue
1600
1601     tbl_cmp_lst = list()
1602     if comparisons:
1603         for row in tbl_lst:
1604             new_row = deepcopy(row)
1605             add_to_tbl = False
1606             for comp in comparisons:
1607                 ref_itm = row[int(comp[u"reference"])]
1608                 if ref_itm is None and \
1609                         comp.get(u"reference-alt", None) is not None:
1610                     ref_itm = row[int(comp[u"reference-alt"])]
1611                 cmp_itm = row[int(comp[u"compare"])]
1612                 if ref_itm is not None and cmp_itm is not None and \
1613                         ref_itm[u"mean"] is not None and \
1614                         cmp_itm[u"mean"] is not None and \
1615                         ref_itm[u"stdev"] is not None and \
1616                         cmp_itm[u"stdev"] is not None:
1617                     delta, d_stdev = relative_change_stdev(
1618                         ref_itm[u"mean"], cmp_itm[u"mean"],
1619                         ref_itm[u"stdev"], cmp_itm[u"stdev"]
1620                     )
1621                     new_row.append(
1622                         {
1623                             u"mean": delta * 1e6,
1624                             u"stdev": d_stdev * 1e6
1625                         }
1626                     )
1627                     add_to_tbl = True
1628                 else:
1629                     new_row.append(None)
1630             if add_to_tbl:
1631                 tbl_cmp_lst.append(new_row)
1632
1633     try:
1634         tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
1635         tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
1636     except TypeError as err:
1637         logging.warning(f"Empty data element in table\n{tbl_cmp_lst}\n{err}")
1638
1639     rcas = list()
1640     rca_in = table.get(u"rca", None)
1641     if rca_in and isinstance(rca_in, list):
1642         for idx, itm in enumerate(rca_in):
1643             try:
1644                 with open(itm.get(u"data", u""), u"r") as rca_file:
1645                     rcas.append(
1646                         {
1647                             u"title": itm.get(u"title", f"RCA{idx}"),
1648                             u"data": load(rca_file, Loader=FullLoader)
1649                         }
1650                     )
1651             except (YAMLError, IOError) as err:
1652                 logging.warning(
1653                     f"The RCA file {itm.get(u'data', u'')} does not exist or "
1654                     f"it is corrupted!"
1655                 )
1656                 logging.debug(repr(err))
1657
1658     tbl_for_csv = list()
1659     for line in tbl_cmp_lst:
1660         row = [line[0], ]
1661         for idx, itm in enumerate(line[1:]):
1662             if itm is None or not isinstance(itm, dict) or\
1663                     itm.get(u'mean', None) is None or \
1664                     itm.get(u'stdev', None) is None:
1665                 row.append(u"NT")
1666                 row.append(u"NT")
1667             else:
1668                 row.append(round(float(itm[u'mean']) / 1e6, 3))
1669                 row.append(round(float(itm[u'stdev']) / 1e6, 3))
1670         for rca in rcas:
1671             rca_nr = rca[u"data"].get(row[0], u"-")
1672             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1673         tbl_for_csv.append(row)
1674
1675     header_csv = [u"Test Case", ]
1676     for col in cols:
1677         header_csv.append(f"Avg({col[u'title']})")
1678         header_csv.append(f"Stdev({col[u'title']})")
1679     for comp in comparisons:
1680         header_csv.append(
1681             f"Avg({comp.get(u'title', u'')})"
1682         )
1683         header_csv.append(
1684             f"Stdev({comp.get(u'title', u'')})"
1685         )
1686     header_csv.extend([rca[u"title"] for rca in rcas])
1687
1688     legend_lst = table.get(u"legend", None)
1689     if legend_lst is None:
1690         legend = u""
1691     else:
1692         legend = u"\n" + u"\n".join(legend_lst) + u"\n"
1693
1694     footnote = u""
1695     if rcas:
1696         footnote += u"\nRCA:\n"
1697         for rca in rcas:
1698             footnote += rca[u"data"].get(u"footnote", u"")
1699
1700     csv_file_name = f"{table[u'output-file']}-csv.csv"
1701     with open(csv_file_name, u"wt", encoding='utf-8') as file_handler:
1702         file_handler.write(
1703             u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
1704         )
1705         for test in tbl_for_csv:
1706             file_handler.write(
1707                 u",".join([f'"{item}"' for item in test]) + u"\n"
1708             )
1709         if legend_lst:
1710             for item in legend_lst:
1711                 file_handler.write(f'"{item}"\n')
1712         if footnote:
1713             for itm in footnote.split(u"\n"):
1714                 file_handler.write(f'"{itm}"\n')
1715
1716     tbl_tmp = list()
1717     max_lens = [0, ] * len(tbl_cmp_lst[0])
1718     for line in tbl_cmp_lst:
1719         row = [line[0], ]
1720         for idx, itm in enumerate(line[1:]):
1721             if itm is None or not isinstance(itm, dict) or \
1722                     itm.get(u'mean', None) is None or \
1723                     itm.get(u'stdev', None) is None:
1724                 new_itm = u"NT"
1725             else:
1726                 if idx < len(cols):
1727                     new_itm = (
1728                         f"{round(float(itm[u'mean']) / 1e6, 1)} "
1729                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1730                         replace(u"nan", u"NaN")
1731                     )
1732                 else:
1733                     new_itm = (
1734                         f"{round(float(itm[u'mean']) / 1e6, 1):+} "
1735                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1736                         replace(u"nan", u"NaN")
1737                     )
1738             if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]:
1739                 max_lens[idx] = len(new_itm.rsplit(u" ", 1)[-1])
1740             row.append(new_itm)
1741
1742         tbl_tmp.append(row)
1743
1744     tbl_final = list()
1745     for line in tbl_tmp:
1746         row = [line[0], ]
1747         for idx, itm in enumerate(line[1:]):
1748             if itm in (u"NT", u"NaN"):
1749                 row.append(itm)
1750                 continue
1751             itm_lst = itm.rsplit(u"\u00B1", 1)
1752             itm_lst[-1] = \
1753                 f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
1754             row.append(u"\u00B1".join(itm_lst))
1755         for rca in rcas:
1756             rca_nr = rca[u"data"].get(row[0], u"-")
1757             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1758
1759         tbl_final.append(row)
1760
1761     header = [u"Test Case", ]
1762     header.extend([col[u"title"] for col in cols])
1763     header.extend([comp.get(u"title", u"") for comp in comparisons])
1764     header.extend([rca[u"title"] for rca in rcas])
1765
1766     # Generate csv tables:
1767     csv_file_name = f"{table[u'output-file']}.csv"
1768     logging.info(f"    Writing the file {csv_file_name}")
1769     with open(csv_file_name, u"wt", encoding='utf-8') as file_handler:
1770         file_handler.write(u";".join(header) + u"\n")
1771         for test in tbl_final:
1772             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1773
1774     # Generate txt table:
1775     txt_file_name = f"{table[u'output-file']}.txt"
1776     logging.info(f"    Writing the file {txt_file_name}")
1777     convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u";")
1778
1779     with open(txt_file_name, u'a', encoding='utf-8') as file_handler:
1780         file_handler.write(legend)
1781         file_handler.write(footnote)
1782
1783     # Generate html table:
1784     _tpc_generate_html_table(
1785         header,
1786         tbl_final,
1787         table[u'output-file'],
1788         legend=legend,
1789         footnote=footnote,
1790         sort_data=False,
1791         title=table.get(u"title", u"")
1792     )
1793
1794
1795 def table_weekly_comparison(table, in_data):
1796     """Generate the table(s) with algorithm: table_weekly_comparison
1797     specified in the specification file.
1798
1799     :param table: Table to generate.
1800     :param in_data: Data to process.
1801     :type table: pandas.Series
1802     :type in_data: InputData
1803     """
1804     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1805
1806     # Transform the data
1807     logging.info(
1808         f"    Creating the data set for the {table.get(u'type', u'')} "
1809         f"{table.get(u'title', u'')}."
1810     )
1811
1812     incl_tests = table.get(u"include-tests", None)
1813     if incl_tests not in (u"NDR", u"PDR"):
1814         logging.error(f"Wrong tests to include specified ({incl_tests}).")
1815         return
1816
1817     nr_cols = table.get(u"nr-of-data-columns", None)
1818     if not nr_cols or nr_cols < 2:
1819         logging.error(
1820             f"No columns specified for {table.get(u'title', u'')}. Skipping."
1821         )
1822         return
1823
1824     data = in_data.filter_data(
1825         table,
1826         params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1827         continue_on_error=True
1828     )
1829
1830     header = [
1831         [u"VPP Version", ],
1832         [u"Start Timestamp", ],
1833         [u"CSIT Build", ],
1834         [u"CSIT Testbed", ]
1835     ]
1836     tbl_dict = dict()
1837     idx = 0
1838     tb_tbl = table.get(u"testbeds", None)
1839     for job_name, job_data in data.items():
1840         for build_nr, build in job_data.items():
1841             if idx >= nr_cols:
1842                 break
1843             if build.empty:
1844                 continue
1845
1846             tb_ip = in_data.metadata(job_name, build_nr).get(u"testbed", u"")
1847             if tb_ip and tb_tbl:
1848                 testbed = tb_tbl.get(tb_ip, u"")
1849             else:
1850                 testbed = u""
1851             header[2].insert(1, build_nr)
1852             header[3].insert(1, testbed)
1853             header[1].insert(
1854                 1, in_data.metadata(job_name, build_nr).get(u"generated", u"")
1855             )
1856             header[0].insert(
1857                 1, in_data.metadata(job_name, build_nr).get(u"version", u"")
1858             )
1859
1860             for tst_name, tst_data in build.items():
1861                 tst_name_mod = \
1862                     _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
1863                 if not tbl_dict.get(tst_name_mod, None):
1864                     tbl_dict[tst_name_mod] = dict(
1865                         name=tst_data[u'name'].rsplit(u'-', 1)[0],
1866                     )
1867                 try:
1868                     tbl_dict[tst_name_mod][-idx - 1] = \
1869                         tst_data[u"throughput"][incl_tests][u"LOWER"]
1870                 except (TypeError, IndexError, KeyError, ValueError):
1871                     pass
1872             idx += 1
1873
1874     if idx < nr_cols:
1875         logging.error(u"Not enough data to build the table! Skipping")
1876         return
1877
1878     cmp_dict = dict()
1879     for idx, cmp in enumerate(table.get(u"comparisons", list())):
1880         idx_ref = cmp.get(u"reference", None)
1881         idx_cmp = cmp.get(u"compare", None)
1882         if idx_ref is None or idx_cmp is None:
1883             continue
1884         header[0].append(
1885             f"Diff({header[0][idx_ref - idx].split(u'~')[-1]} vs "
1886             f"{header[0][idx_cmp - idx].split(u'~')[-1]})"
1887         )
1888         header[1].append(u"")
1889         header[2].append(u"")
1890         header[3].append(u"")
1891         for tst_name, tst_data in tbl_dict.items():
1892             if not cmp_dict.get(tst_name, None):
1893                 cmp_dict[tst_name] = list()
1894             ref_data = tst_data.get(idx_ref, None)
1895             cmp_data = tst_data.get(idx_cmp, None)
1896             if ref_data is None or cmp_data is None:
1897                 cmp_dict[tst_name].append(float('nan'))
1898             else:
1899                 cmp_dict[tst_name].append(
1900                     relative_change(ref_data, cmp_data)
1901                 )
1902
1903     tbl_lst = list()
1904     for tst_name, tst_data in tbl_dict.items():
1905         itm_lst = [tst_data[u"name"], ]
1906         for idx in range(nr_cols):
1907             item = tst_data.get(-idx - 1, None)
1908             if item is None:
1909                 itm_lst.insert(1, None)
1910             else:
1911                 itm_lst.insert(1, round(item / 1e6, 1))
1912         itm_lst.extend(
1913             [
1914                 None if itm is None else round(itm, 1)
1915                 for itm in cmp_dict[tst_name]
1916             ]
1917         )
1918         tbl_lst.append(itm_lst)
1919
1920     tbl_lst.sort(key=lambda rel: rel[0], reverse=False)
1921     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1922
1923     # Generate csv table:
1924     csv_file_name = f"{table[u'output-file']}.csv"
1925     logging.info(f"    Writing the file {csv_file_name}")
1926     with open(csv_file_name, u"wt", encoding='utf-8') as file_handler:
1927         for hdr in header:
1928             file_handler.write(u",".join(hdr) + u"\n")
1929         for test in tbl_lst:
1930             file_handler.write(u",".join(
1931                 [
1932                     str(item).replace(u"None", u"-").replace(u"nan", u"-").
1933                     replace(u"null", u"-") for item in test
1934                 ]
1935             ) + u"\n")
1936
1937     txt_file_name = f"{table[u'output-file']}.txt"
1938     logging.info(f"    Writing the file {txt_file_name}")
1939     convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u",")
1940
1941     # Reorganize header in txt table
1942     txt_table = list()
1943     with open(txt_file_name, u"rt", encoding='utf-8') as file_handler:
1944         for line in file_handler:
1945             txt_table.append(line)
1946     try:
1947         txt_table.insert(5, txt_table.pop(2))
1948         with open(txt_file_name, u"wt", encoding='utf-8') as file_handler:
1949             file_handler.writelines(txt_table)
1950     except IndexError:
1951         pass
1952
1953     # Generate html table:
1954     hdr_html = [
1955         u"<br>".join(row) for row in zip(*header)
1956     ]
1957     _tpc_generate_html_table(
1958         hdr_html,
1959         tbl_lst,
1960         table[u'output-file'],
1961         sort_data=True,
1962         title=table.get(u"title", u""),
1963         generate_rst=False
1964     )