resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from collections import OrderedDict
  23 from xml.etree import ElementTree as ET
  24 from datetime import datetime as dt
  25 from datetime import timedelta
  26 from copy import deepcopy
  27
  28 import plotly.graph_objects as go
  29 import plotly.offline as ploff
  30 import pandas as pd
  31
  32 from numpy import nan, isnan
  33 from yaml import load, FullLoader, YAMLError
  34
  35 from pal_utils import mean, stdev, classify_anomalies, \
  36     convert_csv_to_pretty_txt, relative_change_stdev, relative_change
  37
  38
  39 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
  40
  41
  42 def generate_tables(spec, data):
  43     """Generate all tables specified in the specification file.
  44
  45     :param spec: Specification read from the specification file.
  46     :param data: Data to process.
  47     :type spec: Specification
  48     :type data: InputData
  49     """
  50
  51     generator = {
  52         u"table_merged_details": table_merged_details,
  53         u"table_soak_vs_ndr": table_soak_vs_ndr,
  54         u"table_perf_trending_dash": table_perf_trending_dash,
  55         u"table_perf_trending_dash_html": table_perf_trending_dash_html,
  56         u"table_last_failed_tests": table_last_failed_tests,
  57         u"table_failed_tests": table_failed_tests,
  58         u"table_failed_tests_html": table_failed_tests_html,
  59         u"table_oper_data_html": table_oper_data_html,
  60         u"table_comparison": table_comparison,
  61         u"table_weekly_comparison": table_weekly_comparison
  62     }
  63
  64     logging.info(u"Generating the tables ...")
  65     for table in spec.tables:
  66         try:
  67             if table[u"algorithm"] == u"table_weekly_comparison":
  68                 table[u"testbeds"] = spec.environment.get(u"testbeds", None)
  69             generator[table[u"algorithm"]](table, data)
  70         except NameError as err:
  71             logging.error(
  72                 f"Probably algorithm {table[u'algorithm']} is not defined: "
  73                 f"{repr(err)}"
  74             )
  75     logging.info(u"Done.")
  76
  77
  78 def table_oper_data_html(table, input_data):
  79     """Generate the table(s) with algorithm: html_table_oper_data
  80     specified in the specification file.
  81
  82     :param table: Table to generate.
  83     :param input_data: Data to process.
  84     :type table: pandas.Series
  85     :type input_data: InputData
  86     """
  87
  88     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
  89     # Transform the data
  90     logging.info(
  91         f"    Creating the data set for the {table.get(u'type', u'')} "
  92         f"{table.get(u'title', u'')}."
  93     )
  94     data = input_data.filter_data(
  95         table,
  96         params=[u"name", u"parent", u"show-run", u"type"],
  97         continue_on_error=True
  98     )
  99     if data.empty:
 100         return
 101     data = input_data.merge_data(data)
 102
 103     sort_tests = table.get(u"sort", None)
 104     if sort_tests:
 105         args = dict(
 106             inplace=True,
 107             ascending=(sort_tests == u"ascending")
 108         )
 109         data.sort_index(**args)
 110
 111     suites = input_data.filter_data(
 112         table,
 113         continue_on_error=True,
 114         data_set=u"suites"
 115     )
 116     if suites.empty:
 117         return
 118     suites = input_data.merge_data(suites)
 119
 120     def _generate_html_table(tst_data):
 121         """Generate an HTML table with operational data for the given test.
 122
 123         :param tst_data: Test data to be used to generate the table.
 124         :type tst_data: pandas.Series
 125         :returns: HTML table with operational data.
 126         :rtype: str
 127         """
 128
 129         colors = {
 130             u"header": u"#7eade7",
 131             u"empty": u"#ffffff",
 132             u"body": (u"#e9f1fb", u"#d4e4f7")
 133         }
 134
 135         tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
 136
 137         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
 138         thead = ET.SubElement(
 139             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 140         )
 141         thead.text = tst_data[u"name"]
 142
 143         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 144         thead = ET.SubElement(
 145             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 146         )
 147         thead.text = u"\t"
 148
 149         if tst_data.get(u"show-run", u"No Data") == u"No Data":
 150             trow = ET.SubElement(
 151                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 152             )
 153             tcol = ET.SubElement(
 154                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 155             )
 156             tcol.text = u"No Data"
 157
 158             trow = ET.SubElement(
 159                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 160             )
 161             thead = ET.SubElement(
 162                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 163             )
 164             font = ET.SubElement(
 165                 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 166             )
 167             font.text = u"."
 168             return str(ET.tostring(tbl, encoding=u"unicode"))
 169
 170         tbl_hdr = (
 171             u"Name",
 172             u"Nr of Vectors",
 173             u"Nr of Packets",
 174             u"Suspends",
 175             u"Cycles per Packet",
 176             u"Average Vector Size"
 177         )
 178
 179         for dut_data in tst_data[u"show-run"].values():
 180             trow = ET.SubElement(
 181                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 182             )
 183             tcol = ET.SubElement(
 184                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 185             )
 186             if dut_data.get(u"threads", None) is None:
 187                 tcol.text = u"No Data"
 188                 continue
 189
 190             bold = ET.SubElement(tcol, u"b")
 191             bold.text = (
 192                 f"Host IP: {dut_data.get(u'host', '')}, "
 193                 f"Socket: {dut_data.get(u'socket', '')}"
 194             )
 195             trow = ET.SubElement(
 196                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 197             )
 198             thead = ET.SubElement(
 199                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 200             )
 201             thead.text = u"\t"
 202
 203             for thread_nr, thread in dut_data[u"threads"].items():
 204                 trow = ET.SubElement(
 205                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 206                 )
 207                 tcol = ET.SubElement(
 208                     trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 209                 )
 210                 bold = ET.SubElement(tcol, u"b")
 211                 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
 212                 trow = ET.SubElement(
 213                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 214                 )
 215                 for idx, col in enumerate(tbl_hdr):
 216                     tcol = ET.SubElement(
 217                         trow, u"td",
 218                         attrib=dict(align=u"right" if idx else u"left")
 219                     )
 220                     font = ET.SubElement(
 221                         tcol, u"font", attrib=dict(size=u"2")
 222                     )
 223                     bold = ET.SubElement(font, u"b")
 224                     bold.text = col
 225                 for row_nr, row in enumerate(thread):
 226                     trow = ET.SubElement(
 227                         tbl, u"tr",
 228                         attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
 229                     )
 230                     for idx, col in enumerate(row):
 231                         tcol = ET.SubElement(
 232                             trow, u"td",
 233                             attrib=dict(align=u"right" if idx else u"left")
 234                         )
 235                         font = ET.SubElement(
 236                             tcol, u"font", attrib=dict(size=u"2")
 237                         )
 238                         if isinstance(col, float):
 239                             font.text = f"{col:.2f}"
 240                         else:
 241                             font.text = str(col)
 242                 trow = ET.SubElement(
 243                     tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 244                 )
 245                 thead = ET.SubElement(
 246                     trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 247                 )
 248                 thead.text = u"\t"
 249
 250         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 251         thead = ET.SubElement(
 252             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 253         )
 254         font = ET.SubElement(
 255             thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 256         )
 257         font.text = u"."
 258
 259         return str(ET.tostring(tbl, encoding=u"unicode"))
 260
 261     for suite in suites.values:
 262         html_table = str()
 263         for test_data in data.values:
 264             if test_data[u"parent"] not in suite[u"name"]:
 265                 continue
 266             html_table += _generate_html_table(test_data)
 267         if not html_table:
 268             continue
 269         try:
 270             file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
 271             with open(f"{file_name}", u'w') as html_file:
 272                 logging.info(f"    Writing file: {file_name}")
 273                 html_file.write(u".. raw:: html\n\n\t")
 274                 html_file.write(html_table)
 275                 html_file.write(u"\n\t<p><br><br></p>\n")
 276         except KeyError:
 277             logging.warning(u"The output file is not defined.")
 278             return
 279     logging.info(u"  Done.")
 280
 281
 282 def table_merged_details(table, input_data):
 283     """Generate the table(s) with algorithm: table_merged_details
 284     specified in the specification file.
 285
 286     :param table: Table to generate.
 287     :param input_data: Data to process.
 288     :type table: pandas.Series
 289     :type input_data: InputData
 290     """
 291
 292     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 293
 294     # Transform the data
 295     logging.info(
 296         f"    Creating the data set for the {table.get(u'type', u'')} "
 297         f"{table.get(u'title', u'')}."
 298     )
 299     data = input_data.filter_data(table, continue_on_error=True)
 300     data = input_data.merge_data(data)
 301
 302     sort_tests = table.get(u"sort", None)
 303     if sort_tests:
 304         args = dict(
 305             inplace=True,
 306             ascending=(sort_tests == u"ascending")
 307         )
 308         data.sort_index(**args)
 309
 310     suites = input_data.filter_data(
 311         table, continue_on_error=True, data_set=u"suites")
 312     suites = input_data.merge_data(suites)
 313
 314     # Prepare the header of the tables
 315     header = list()
 316     for column in table[u"columns"]:
 317         header.append(
 318             u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
 319         )
 320
 321     for suite in suites.values:
 322         # Generate data
 323         suite_name = suite[u"name"]
 324         table_lst = list()
 325         for test in data.keys():
 326             if data[test][u"parent"] not in suite_name:
 327                 continue
 328             row_lst = list()
 329             for column in table[u"columns"]:
 330                 try:
 331                     col_data = str(data[test][column[
 332                         u"data"].split(u" ")[1]]).replace(u'"', u'""')
 333                     # Do not include tests with "Test Failed" in test message
 334                     if u"Test Failed" in col_data:
 335                         continue
 336                     col_data = col_data.replace(
 337                         u"No Data", u"Not Captured     "
 338                     )
 339                     if column[u"data"].split(u" ")[1] in (u"name", ):
 340                         if len(col_data) > 30:
 341                             col_data_lst = col_data.split(u"-")
 342                             half = int(len(col_data_lst) / 2)
 343                             col_data = f"{u'-'.join(col_data_lst[:half])}" \
 344                                        f"- |br| " \
 345                                        f"{u'-'.join(col_data_lst[half:])}"
 346                         col_data = f" |prein| {col_data} |preout| "
 347                     elif column[u"data"].split(u" ")[1] in (u"msg", ):
 348                         # Temporary solution: remove NDR results from message:
 349                         if bool(table.get(u'remove-ndr', False)):
 350                             try:
 351                                 col_data = col_data.split(u" |br| ", 1)[1]
 352                             except IndexError:
 353                                 pass
 354                         col_data = f" |prein| {col_data} |preout| "
 355                     elif column[u"data"].split(u" ")[1] in \
 356                             (u"conf-history", u"show-run"):
 357                         col_data = col_data.replace(u" |br| ", u"", 1)
 358                         col_data = f" |prein| {col_data[:-5]} |preout| "
 359                     row_lst.append(f'"{col_data}"')
 360                 except KeyError:
 361                     row_lst.append(u'"Not captured"')
 362             if len(row_lst) == len(table[u"columns"]):
 363                 table_lst.append(row_lst)
 364
 365         # Write the data to file
 366         if table_lst:
 367             separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
 368             file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
 369             logging.info(f"      Writing file: {file_name}")
 370             with open(file_name, u"wt") as file_handler:
 371                 file_handler.write(u",".join(header) + u"\n")
 372                 for item in table_lst:
 373                     file_handler.write(u",".join(item) + u"\n")
 374
 375     logging.info(u"  Done.")
 376
 377
 378 def _tpc_modify_test_name(test_name, ignore_nic=False):
 379     """Modify a test name by replacing its parts.
 380
 381     :param test_name: Test name to be modified.
 382     :param ignore_nic: If True, NIC is removed from TC name.
 383     :type test_name: str
 384     :type ignore_nic: bool
 385     :returns: Modified test name.
 386     :rtype: str
 387     """
 388     test_name_mod = test_name.\
 389         replace(u"-ndrpdrdisc", u""). \
 390         replace(u"-ndrpdr", u"").\
 391         replace(u"-pdrdisc", u""). \
 392         replace(u"-ndrdisc", u"").\
 393         replace(u"-pdr", u""). \
 394         replace(u"-ndr", u""). \
 395         replace(u"1t1c", u"1c").\
 396         replace(u"2t1c", u"1c"). \
 397         replace(u"2t2c", u"2c").\
 398         replace(u"4t2c", u"2c"). \
 399         replace(u"4t4c", u"4c").\
 400         replace(u"8t4c", u"4c")
 401
 402     if ignore_nic:
 403         return re.sub(REGEX_NIC, u"", test_name_mod)
 404     return test_name_mod
 405
 406
 407 def _tpc_modify_displayed_test_name(test_name):
 408     """Modify a test name which is displayed in a table by replacing its parts.
 409
 410     :param test_name: Test name to be modified.
 411     :type test_name: str
 412     :returns: Modified test name.
 413     :rtype: str
 414     """
 415     return test_name.\
 416         replace(u"1t1c", u"1c").\
 417         replace(u"2t1c", u"1c"). \
 418         replace(u"2t2c", u"2c").\
 419         replace(u"4t2c", u"2c"). \
 420         replace(u"4t4c", u"4c").\
 421         replace(u"8t4c", u"4c")
 422
 423
 424 def _tpc_insert_data(target, src, include_tests):
 425     """Insert src data to the target structure.
 426
 427     :param target: Target structure where the data is placed.
 428     :param src: Source data to be placed into the target stucture.
 429     :param include_tests: Which results will be included (MRR, NDR, PDR).
 430     :type target: list
 431     :type src: dict
 432     :type include_tests: str
 433     """
 434     try:
 435         if include_tests == u"MRR":
 436             target[u"mean"] = src[u"result"][u"receive-rate"]
 437             target[u"stdev"] = src[u"result"][u"receive-stdev"]
 438         elif include_tests == u"PDR":
 439             target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
 440         elif include_tests == u"NDR":
 441             target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
 442     except (KeyError, TypeError):
 443         pass
 444
 445
 446 def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
 447                              footnote=u"", sort_data=True, title=u"",
 448                              generate_rst=True):
 449     """Generate html table from input data with simple sorting possibility.
 450
 451     :param header: Table header.
 452     :param data: Input data to be included in the table. It is a list of lists.
 453         Inner lists are rows in the table. All inner lists must be of the same
 454         length. The length of these lists must be the same as the length of the
 455         header.
 456     :param out_file_name: The name (relative or full path) where the
 457         generated html table is written.
 458     :param legend: The legend to display below the table.
 459     :param footnote: The footnote to display below the table (and legend).
 460     :param sort_data: If True the data sorting is enabled.
 461     :param title: The table (and file) title.
 462     :param generate_rst: If True, wrapping rst file is generated.
 463     :type header: list
 464     :type data: list of lists
 465     :type out_file_name: str
 466     :type legend: str
 467     :type footnote: str
 468     :type sort_data: bool
 469     :type title: str
 470     :type generate_rst: bool
 471     """
 472
 473     try:
 474         idx = header.index(u"Test Case")
 475     except ValueError:
 476         idx = 0
 477     params = {
 478         u"align-hdr": (
 479             [u"left", u"right"],
 480             [u"left", u"left", u"right"],
 481             [u"left", u"left", u"left", u"right"]
 482         ),
 483         u"align-itm": (
 484             [u"left", u"right"],
 485             [u"left", u"left", u"right"],
 486             [u"left", u"left", u"left", u"right"]
 487         ),
 488         u"width": ([15, 9], [4, 24, 10], [4, 4, 32, 10])
 489     }
 490
 491     df_data = pd.DataFrame(data, columns=header)
 492
 493     if sort_data:
 494         df_sorted = [df_data.sort_values(
 495             by=[key, header[idx]], ascending=[True, True]
 496             if key != header[idx] else [False, True]) for key in header]
 497         df_sorted_rev = [df_data.sort_values(
 498             by=[key, header[idx]], ascending=[False, True]
 499             if key != header[idx] else [True, True]) for key in header]
 500         df_sorted.extend(df_sorted_rev)
 501     else:
 502         df_sorted = df_data
 503
 504     fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
 505                    for idx in range(len(df_data))]]
 506     table_header = dict(
 507         values=[f"<b>{item.replace(u',', u',<br>')}</b>" for item in header],
 508         fill_color=u"#7eade7",
 509         align=params[u"align-hdr"][idx],
 510         font=dict(
 511             family=u"Courier New",
 512             size=12
 513         )
 514     )
 515
 516     fig = go.Figure()
 517
 518     if sort_data:
 519         for table in df_sorted:
 520             columns = [table.get(col) for col in header]
 521             fig.add_trace(
 522                 go.Table(
 523                     columnwidth=params[u"width"][idx],
 524                     header=table_header,
 525                     cells=dict(
 526                         values=columns,
 527                         fill_color=fill_color,
 528                         align=params[u"align-itm"][idx],
 529                         font=dict(
 530                             family=u"Courier New",
 531                             size=12
 532                         )
 533                     )
 534                 )
 535             )
 536
 537         buttons = list()
 538         menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
 539         menu_items.extend([f"<b>{itm}</b> (descending)" for itm in header])
 540         for idx, hdr in enumerate(menu_items):
 541             visible = [False, ] * len(menu_items)
 542             visible[idx] = True
 543             buttons.append(
 544                 dict(
 545                     label=hdr.replace(u" [Mpps]", u""),
 546                     method=u"update",
 547                     args=[{u"visible": visible}],
 548                 )
 549             )
 550
 551         fig.update_layout(
 552             updatemenus=[
 553                 go.layout.Updatemenu(
 554                     type=u"dropdown",
 555                     direction=u"down",
 556                     x=0.0,
 557                     xanchor=u"left",
 558                     y=1.002,
 559                     yanchor=u"bottom",
 560                     active=len(menu_items) - 1,
 561                     buttons=list(buttons)
 562                 )
 563             ],
 564         )
 565     else:
 566         fig.add_trace(
 567             go.Table(
 568                 columnwidth=params[u"width"][idx],
 569                 header=table_header,
 570                 cells=dict(
 571                     values=[df_sorted.get(col) for col in header],
 572                     fill_color=fill_color,
 573                     align=params[u"align-itm"][idx],
 574                     font=dict(
 575                         family=u"Courier New",
 576                         size=12
 577                     )
 578                 )
 579             )
 580         )
 581
 582     ploff.plot(
 583         fig,
 584         show_link=False,
 585         auto_open=False,
 586         filename=f"{out_file_name}_in.html"
 587     )
 588
 589     if not generate_rst:
 590         return
 591
 592     file_name = out_file_name.split(u"/")[-1]
 593     if u"vpp" in out_file_name:
 594         path = u"_tmp/src/vpp_performance_tests/comparisons/"
 595     else:
 596         path = u"_tmp/src/dpdk_performance_tests/comparisons/"
 597     with open(f"{path}{file_name}.rst", u"wt") as rst_file:
 598         rst_file.write(
 599             u"\n"
 600             u".. |br| raw:: html\n\n    <br />\n\n\n"
 601             u".. |prein| raw:: html\n\n    <pre>\n\n\n"
 602             u".. |preout| raw:: html\n\n    </pre>\n\n"
 603         )
 604         if title:
 605             rst_file.write(f"{title}\n")
 606             rst_file.write(f"{u'`' * len(title)}\n\n")
 607         rst_file.write(
 608             u".. raw:: html\n\n"
 609             f'    <iframe frameborder="0" scrolling="no" '
 610             f'width="1600" height="1200" '
 611             f'src="../..{out_file_name.replace(u"_build", u"")}_in.html">'
 612             f'</iframe>\n\n'
 613         )
 614         if legend:
 615             rst_file.write(legend[1:].replace(u"\n", u" |br| "))
 616         if footnote:
 617             rst_file.write(footnote.replace(u"\n", u" |br| ")[1:])
 618
 619
 620 def table_soak_vs_ndr(table, input_data):
 621     """Generate the table(s) with algorithm: table_soak_vs_ndr
 622     specified in the specification file.
 623
 624     :param table: Table to generate.
 625     :param input_data: Data to process.
 626     :type table: pandas.Series
 627     :type input_data: InputData
 628     """
 629
 630     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 631
 632     # Transform the data
 633     logging.info(
 634         f"    Creating the data set for the {table.get(u'type', u'')} "
 635         f"{table.get(u'title', u'')}."
 636     )
 637     data = input_data.filter_data(table, continue_on_error=True)
 638
 639     # Prepare the header of the table
 640     try:
 641         header = [
 642             u"Test Case",
 643             f"Avg({table[u'reference'][u'title']})",
 644             f"Stdev({table[u'reference'][u'title']})",
 645             f"Avg({table[u'compare'][u'title']})",
 646             f"Stdev{table[u'compare'][u'title']})",
 647             u"Diff",
 648             u"Stdev(Diff)"
 649         ]
 650         header_str = u";".join(header) + u"\n"
 651         legend = (
 652             u"\nLegend:\n"
 653             f"Avg({table[u'reference'][u'title']}): "
 654             f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
 655             f"from a series of runs of the listed tests.\n"
 656             f"Stdev({table[u'reference'][u'title']}): "
 657             f"Standard deviation value of {table[u'reference'][u'title']} "
 658             f"[Mpps] computed from a series of runs of the listed tests.\n"
 659             f"Avg({table[u'compare'][u'title']}): "
 660             f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
 661             f"a series of runs of the listed tests.\n"
 662             f"Stdev({table[u'compare'][u'title']}): "
 663             f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
 664             f"computed from a series of runs of the listed tests.\n"
 665             f"Diff({table[u'reference'][u'title']},"
 666             f"{table[u'compare'][u'title']}): "
 667             f"Percentage change calculated for mean values.\n"
 668             u"Stdev(Diff): "
 669             u"Standard deviation of percentage change calculated for mean "
 670             u"values.\n"
 671             u":END"
 672         )
 673     except (AttributeError, KeyError) as err:
 674         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 675         return
 676
 677     # Create a list of available SOAK test results:
 678     tbl_dict = dict()
 679     for job, builds in table[u"compare"][u"data"].items():
 680         for build in builds:
 681             for tst_name, tst_data in data[job][str(build)].items():
 682                 if tst_data[u"type"] == u"SOAK":
 683                     tst_name_mod = tst_name.replace(u"-soak", u"")
 684                     if tbl_dict.get(tst_name_mod, None) is None:
 685                         groups = re.search(REGEX_NIC, tst_data[u"parent"])
 686                         nic = groups.group(0) if groups else u""
 687                         name = (
 688                             f"{nic}-"
 689                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 690                         )
 691                         tbl_dict[tst_name_mod] = {
 692                             u"name": name,
 693                             u"ref-data": list(),
 694                             u"cmp-data": list()
 695                         }
 696                     try:
 697                         tbl_dict[tst_name_mod][u"cmp-data"].append(
 698                             tst_data[u"throughput"][u"LOWER"])
 699                     except (KeyError, TypeError):
 700                         pass
 701     tests_lst = tbl_dict.keys()
 702
 703     # Add corresponding NDR test results:
 704     for job, builds in table[u"reference"][u"data"].items():
 705         for build in builds:
 706             for tst_name, tst_data in data[job][str(build)].items():
 707                 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
 708                     replace(u"-mrr", u"")
 709                 if tst_name_mod not in tests_lst:
 710                     continue
 711                 try:
 712                     if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
 713                         continue
 714                     if table[u"include-tests"] == u"MRR":
 715                         result = (tst_data[u"result"][u"receive-rate"],
 716                                   tst_data[u"result"][u"receive-stdev"])
 717                     elif table[u"include-tests"] == u"PDR":
 718                         result = \
 719                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
 720                     elif table[u"include-tests"] == u"NDR":
 721                         result = \
 722                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
 723                     else:
 724                         result = None
 725                     if result is not None:
 726                         tbl_dict[tst_name_mod][u"ref-data"].append(
 727                             result)
 728                 except (KeyError, TypeError):
 729                     continue
 730
 731     tbl_lst = list()
 732     for tst_name in tbl_dict:
 733         item = [tbl_dict[tst_name][u"name"], ]
 734         data_r = tbl_dict[tst_name][u"ref-data"]
 735         if data_r:
 736             if table[u"include-tests"] == u"MRR":
 737                 data_r_mean = data_r[0][0]
 738                 data_r_stdev = data_r[0][1]
 739             else:
 740                 data_r_mean = mean(data_r)
 741                 data_r_stdev = stdev(data_r)
 742             item.append(round(data_r_mean / 1e6, 1))
 743             item.append(round(data_r_stdev / 1e6, 1))
 744         else:
 745             data_r_mean = None
 746             data_r_stdev = None
 747             item.extend([None, None])
 748         data_c = tbl_dict[tst_name][u"cmp-data"]
 749         if data_c:
 750             if table[u"include-tests"] == u"MRR":
 751                 data_c_mean = data_c[0][0]
 752                 data_c_stdev = data_c[0][1]
 753             else:
 754                 data_c_mean = mean(data_c)
 755                 data_c_stdev = stdev(data_c)
 756             item.append(round(data_c_mean / 1e6, 1))
 757             item.append(round(data_c_stdev / 1e6, 1))
 758         else:
 759             data_c_mean = None
 760             data_c_stdev = None
 761             item.extend([None, None])
 762         if data_r_mean is not None and data_c_mean is not None:
 763             delta, d_stdev = relative_change_stdev(
 764                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
 765             try:
 766                 item.append(round(delta))
 767             except ValueError:
 768                 item.append(delta)
 769             try:
 770                 item.append(round(d_stdev))
 771             except ValueError:
 772                 item.append(d_stdev)
 773             tbl_lst.append(item)
 774
 775     # Sort the table according to the relative change
 776     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 777
 778     # Generate csv tables:
 779     csv_file = f"{table[u'output-file']}.csv"
 780     with open(csv_file, u"wt") as file_handler:
 781         file_handler.write(header_str)
 782         for test in tbl_lst:
 783             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 784
 785     convert_csv_to_pretty_txt(
 786         csv_file, f"{table[u'output-file']}.txt", delimiter=u";"
 787     )
 788     with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
 789         txt_file.write(legend)
 790
 791     # Generate html table:
 792     _tpc_generate_html_table(
 793         header,
 794         tbl_lst,
 795         table[u'output-file'],
 796         legend=legend,
 797         title=table.get(u"title", u"")
 798     )
 799
 800
 801 def table_perf_trending_dash(table, input_data):
 802     """Generate the table(s) with algorithm:
 803     table_perf_trending_dash
 804     specified in the specification file.
 805
 806     :param table: Table to generate.
 807     :param input_data: Data to process.
 808     :type table: pandas.Series
 809     :type input_data: InputData
 810     """
 811
 812     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 813
 814     # Transform the data
 815     logging.info(
 816         f"    Creating the data set for the {table.get(u'type', u'')} "
 817         f"{table.get(u'title', u'')}."
 818     )
 819     data = input_data.filter_data(table, continue_on_error=True)
 820
 821     # Prepare the header of the tables
 822     header = [
 823         u"Test Case",
 824         u"Trend [Mpps]",
 825         u"Short-Term Change [%]",
 826         u"Long-Term Change [%]",
 827         u"Regressions [#]",
 828         u"Progressions [#]"
 829     ]
 830     header_str = u",".join(header) + u"\n"
 831
 832     incl_tests = table.get(u"include-tests", u"MRR")
 833
 834     # Prepare data to the table:
 835     tbl_dict = dict()
 836     for job, builds in table[u"data"].items():
 837         for build in builds:
 838             for tst_name, tst_data in data[job][str(build)].items():
 839                 if tst_name.lower() in table.get(u"ignore-list", list()):
 840                     continue
 841                 if tbl_dict.get(tst_name, None) is None:
 842                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
 843                     if not groups:
 844                         continue
 845                     nic = groups.group(0)
 846                     tbl_dict[tst_name] = {
 847                         u"name": f"{nic}-{tst_data[u'name']}",
 848                         u"data": OrderedDict()
 849                     }
 850                 try:
 851                     if incl_tests == u"MRR":
 852                         tbl_dict[tst_name][u"data"][str(build)] = \
 853                             tst_data[u"result"][u"receive-rate"]
 854                     elif incl_tests == u"NDR":
 855                         tbl_dict[tst_name][u"data"][str(build)] = \
 856                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
 857                     elif incl_tests == u"PDR":
 858                         tbl_dict[tst_name][u"data"][str(build)] = \
 859                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
 860                 except (TypeError, KeyError):
 861                     pass  # No data in output.xml for this test
 862
 863     tbl_lst = list()
 864     for tst_name in tbl_dict:
 865         data_t = tbl_dict[tst_name][u"data"]
 866         if len(data_t) < 2:
 867             continue
 868
 869         classification_lst, avgs, _ = classify_anomalies(data_t)
 870
 871         win_size = min(len(data_t), table[u"window"])
 872         long_win_size = min(len(data_t), table[u"long-trend-window"])
 873
 874         try:
 875             max_long_avg = max(
 876                 [x for x in avgs[-long_win_size:-win_size]
 877                  if not isnan(x)])
 878         except ValueError:
 879             max_long_avg = nan
 880         last_avg = avgs[-1]
 881         avg_week_ago = avgs[max(-win_size, -len(avgs))]
 882
 883         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
 884             rel_change_last = nan
 885         else:
 886             rel_change_last = round(
 887                 ((last_avg - avg_week_ago) / avg_week_ago) * 1e2, 2)
 888
 889         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
 890             rel_change_long = nan
 891         else:
 892             rel_change_long = round(
 893                 ((last_avg - max_long_avg) / max_long_avg) * 1e2, 2)
 894
 895         if classification_lst:
 896             if isnan(rel_change_last) and isnan(rel_change_long):
 897                 continue
 898             if isnan(last_avg) or isnan(rel_change_last) or \
 899                     isnan(rel_change_long):
 900                 continue
 901             tbl_lst.append(
 902                 [tbl_dict[tst_name][u"name"],
 903                  round(last_avg / 1e6, 2),
 904                  rel_change_last,
 905                  rel_change_long,
 906                  classification_lst[-win_size+1:].count(u"regression"),
 907                  classification_lst[-win_size+1:].count(u"progression")])
 908
 909     tbl_lst.sort(key=lambda rel: rel[0])
 910
 911     tbl_sorted = list()
 912     for nrr in range(table[u"window"], -1, -1):
 913         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
 914         for nrp in range(table[u"window"], -1, -1):
 915             tbl_out = [item for item in tbl_reg if item[5] == nrp]
 916             tbl_out.sort(key=lambda rel: rel[2])
 917             tbl_sorted.extend(tbl_out)
 918
 919     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
 920
 921     logging.info(f"    Writing file: {file_name}")
 922     with open(file_name, u"wt") as file_handler:
 923         file_handler.write(header_str)
 924         for test in tbl_sorted:
 925             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
 926
 927     logging.info(f"    Writing file: {table[u'output-file']}.txt")
 928     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
 929
 930
 931 def _generate_url(testbed, test_name):
 932     """Generate URL to a trending plot from the name of the test case.
 933
 934     :param testbed: The testbed used for testing.
 935     :param test_name: The name of the test case.
 936     :type testbed: str
 937     :type test_name: str
 938     :returns: The URL to the plot with the trending data for the given test
 939         case.
 940     :rtype str
 941     """
 942
 943     if u"x520" in test_name:
 944         nic = u"x520"
 945     elif u"x710" in test_name:
 946         nic = u"x710"
 947     elif u"xl710" in test_name:
 948         nic = u"xl710"
 949     elif u"xxv710" in test_name:
 950         nic = u"xxv710"
 951     elif u"vic1227" in test_name:
 952         nic = u"vic1227"
 953     elif u"vic1385" in test_name:
 954         nic = u"vic1385"
 955     elif u"x553" in test_name:
 956         nic = u"x553"
 957     else:
 958         nic = u""
 959
 960     if u"64b" in test_name:
 961         frame_size = u"64b"
 962     elif u"78b" in test_name:
 963         frame_size = u"78b"
 964     elif u"imix" in test_name:
 965         frame_size = u"imix"
 966     elif u"9000b" in test_name:
 967         frame_size = u"9000b"
 968     elif u"1518b" in test_name:
 969         frame_size = u"1518b"
 970     elif u"114b" in test_name:
 971         frame_size = u"114b"
 972     else:
 973         frame_size = u""
 974
 975     if u"1t1c" in test_name or \
 976         (u"-1c-" in test_name and
 977          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 978         cores = u"1t1c"
 979     elif u"2t2c" in test_name or \
 980          (u"-2c-" in test_name and
 981           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 982         cores = u"2t2c"
 983     elif u"4t4c" in test_name or \
 984          (u"-4c-" in test_name and
 985           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 986         cores = u"4t4c"
 987     elif u"2t1c" in test_name or \
 988          (u"-1c-" in test_name and
 989           testbed in (u"2n-skx", u"3n-skx")):
 990         cores = u"2t1c"
 991     elif u"4t2c" in test_name:
 992         cores = u"4t2c"
 993     elif u"8t4c" in test_name:
 994         cores = u"8t4c"
 995     else:
 996         cores = u""
 997
 998     if u"testpmd" in test_name:
 999         driver = u"testpmd"
1000     elif u"l3fwd" in test_name:
1001         driver = u"l3fwd"
1002     elif u"avf" in test_name:
1003         driver = u"avf"
1004     elif u"dnv" in testbed or u"tsh" in testbed:
1005         driver = u"ixgbe"
1006     else:
1007         driver = u"dpdk"
1008
1009     if u"acl" in test_name or \
1010             u"macip" in test_name or \
1011             u"nat" in test_name or \
1012             u"policer" in test_name or \
1013             u"cop" in test_name:
1014         bsf = u"features"
1015     elif u"scale" in test_name:
1016         bsf = u"scale"
1017     elif u"base" in test_name:
1018         bsf = u"base"
1019     else:
1020         bsf = u"base"
1021
1022     if u"114b" in test_name and u"vhost" in test_name:
1023         domain = u"vts"
1024     elif u"testpmd" in test_name or u"l3fwd" in test_name:
1025         domain = u"dpdk"
1026     elif u"memif" in test_name:
1027         domain = u"container_memif"
1028     elif u"srv6" in test_name:
1029         domain = u"srv6"
1030     elif u"vhost" in test_name:
1031         domain = u"vhost"
1032         if u"vppl2xc" in test_name:
1033             driver += u"-vpp"
1034         else:
1035             driver += u"-testpmd"
1036         if u"lbvpplacp" in test_name:
1037             bsf += u"-link-bonding"
1038     elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1039         domain = u"nf_service_density_vnfc"
1040     elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1041         domain = u"nf_service_density_cnfc"
1042     elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1043         domain = u"nf_service_density_cnfp"
1044     elif u"ipsec" in test_name:
1045         domain = u"ipsec"
1046         if u"sw" in test_name:
1047             bsf += u"-sw"
1048         elif u"hw" in test_name:
1049             bsf += u"-hw"
1050     elif u"ethip4vxlan" in test_name:
1051         domain = u"ip4_tunnels"
1052     elif u"ip4base" in test_name or u"ip4scale" in test_name:
1053         domain = u"ip4"
1054     elif u"ip6base" in test_name or u"ip6scale" in test_name:
1055         domain = u"ip6"
1056     elif u"l2xcbase" in test_name or \
1057             u"l2xcscale" in test_name or \
1058             u"l2bdbasemaclrn" in test_name or \
1059             u"l2bdscale" in test_name or \
1060             u"l2patch" in test_name:
1061         domain = u"l2"
1062     else:
1063         domain = u""
1064
1065     file_name = u"-".join((domain, testbed, nic)) + u".html#"
1066     anchor_name = u"-".join((frame_size, cores, bsf, driver))
1067
1068     return file_name + anchor_name
1069
1070
1071 def table_perf_trending_dash_html(table, input_data):
1072     """Generate the table(s) with algorithm:
1073     table_perf_trending_dash_html specified in the specification
1074     file.
1075
1076     :param table: Table to generate.
1077     :param input_data: Data to process.
1078     :type table: dict
1079     :type input_data: InputData
1080     """
1081
1082     _ = input_data
1083
1084     if not table.get(u"testbed", None):
1085         logging.error(
1086             f"The testbed is not defined for the table "
1087             f"{table.get(u'title', u'')}."
1088         )
1089         return
1090
1091     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1092
1093     try:
1094         with open(table[u"input-file"], u'rt') as csv_file:
1095             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1096     except KeyError:
1097         logging.warning(u"The input file is not defined.")
1098         return
1099     except csv.Error as err:
1100         logging.warning(
1101             f"Not possible to process the file {table[u'input-file']}.\n"
1102             f"{repr(err)}"
1103         )
1104         return
1105
1106     # Table:
1107     dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1108
1109     # Table header:
1110     trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1111     for idx, item in enumerate(csv_lst[0]):
1112         alignment = u"left" if idx == 0 else u"center"
1113         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1114         thead.text = item
1115
1116     # Rows:
1117     colors = {
1118         u"regression": (
1119             u"#ffcccc",
1120             u"#ff9999"
1121         ),
1122         u"progression": (
1123             u"#c6ecc6",
1124             u"#9fdf9f"
1125         ),
1126         u"normal": (
1127             u"#e9f1fb",
1128             u"#d4e4f7"
1129         )
1130     }
1131     for r_idx, row in enumerate(csv_lst[1:]):
1132         if int(row[4]):
1133             color = u"regression"
1134         elif int(row[5]):
1135             color = u"progression"
1136         else:
1137             color = u"normal"
1138         trow = ET.SubElement(
1139             dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
1140         )
1141
1142         # Columns:
1143         for c_idx, item in enumerate(row):
1144             tdata = ET.SubElement(
1145                 trow,
1146                 u"td",
1147                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1148             )
1149             # Name:
1150             if c_idx == 0 and table.get(u"add-links", True):
1151                 ref = ET.SubElement(
1152                     tdata,
1153                     u"a",
1154                     attrib=dict(
1155                         href=f"../trending/"
1156                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1157                     )
1158                 )
1159                 ref.text = item
1160             else:
1161                 tdata.text = item
1162     try:
1163         with open(table[u"output-file"], u'w') as html_file:
1164             logging.info(f"    Writing file: {table[u'output-file']}")
1165             html_file.write(u".. raw:: html\n\n\t")
1166             html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
1167             html_file.write(u"\n\t<p><br><br></p>\n")
1168     except KeyError:
1169         logging.warning(u"The output file is not defined.")
1170         return
1171
1172
1173 def table_last_failed_tests(table, input_data):
1174     """Generate the table(s) with algorithm: table_last_failed_tests
1175     specified in the specification file.
1176
1177     :param table: Table to generate.
1178     :param input_data: Data to process.
1179     :type table: pandas.Series
1180     :type input_data: InputData
1181     """
1182
1183     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1184
1185     # Transform the data
1186     logging.info(
1187         f"    Creating the data set for the {table.get(u'type', u'')} "
1188         f"{table.get(u'title', u'')}."
1189     )
1190
1191     data = input_data.filter_data(table, continue_on_error=True)
1192
1193     if data is None or data.empty:
1194         logging.warning(
1195             f"    No data for the {table.get(u'type', u'')} "
1196             f"{table.get(u'title', u'')}."
1197         )
1198         return
1199
1200     tbl_list = list()
1201     for job, builds in table[u"data"].items():
1202         for build in builds:
1203             build = str(build)
1204             try:
1205                 version = input_data.metadata(job, build).get(u"version", u"")
1206             except KeyError:
1207                 logging.error(f"Data for {job}: {build} is not present.")
1208                 return
1209             tbl_list.append(build)
1210             tbl_list.append(version)
1211             failed_tests = list()
1212             passed = 0
1213             failed = 0
1214             for tst_data in data[job][build].values:
1215                 if tst_data[u"status"] != u"FAIL":
1216                     passed += 1
1217                     continue
1218                 failed += 1
1219                 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1220                 if not groups:
1221                     continue
1222                 nic = groups.group(0)
1223                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
1224             tbl_list.append(str(passed))
1225             tbl_list.append(str(failed))
1226             tbl_list.extend(failed_tests)
1227
1228     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1229     logging.info(f"    Writing file: {file_name}")
1230     with open(file_name, u"wt") as file_handler:
1231         for test in tbl_list:
1232             file_handler.write(test + u'\n')
1233
1234
1235 def table_failed_tests(table, input_data):
1236     """Generate the table(s) with algorithm: table_failed_tests
1237     specified in the specification file.
1238
1239     :param table: Table to generate.
1240     :param input_data: Data to process.
1241     :type table: pandas.Series
1242     :type input_data: InputData
1243     """
1244
1245     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1246
1247     # Transform the data
1248     logging.info(
1249         f"    Creating the data set for the {table.get(u'type', u'')} "
1250         f"{table.get(u'title', u'')}."
1251     )
1252     data = input_data.filter_data(table, continue_on_error=True)
1253
1254     # Prepare the header of the tables
1255     header = [
1256         u"Test Case",
1257         u"Failures [#]",
1258         u"Last Failure [Time]",
1259         u"Last Failure [VPP-Build-Id]",
1260         u"Last Failure [CSIT-Job-Build-Id]"
1261     ]
1262
1263     # Generate the data for the table according to the model in the table
1264     # specification
1265
1266     now = dt.utcnow()
1267     timeperiod = timedelta(int(table.get(u"window", 7)))
1268
1269     tbl_dict = dict()
1270     for job, builds in table[u"data"].items():
1271         for build in builds:
1272             build = str(build)
1273             for tst_name, tst_data in data[job][build].items():
1274                 if tst_name.lower() in table.get(u"ignore-list", list()):
1275                     continue
1276                 if tbl_dict.get(tst_name, None) is None:
1277                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1278                     if not groups:
1279                         continue
1280                     nic = groups.group(0)
1281                     tbl_dict[tst_name] = {
1282                         u"name": f"{nic}-{tst_data[u'name']}",
1283                         u"data": OrderedDict()
1284                     }
1285                 try:
1286                     generated = input_data.metadata(job, build).\
1287                         get(u"generated", u"")
1288                     if not generated:
1289                         continue
1290                     then = dt.strptime(generated, u"%Y%m%d %H:%M")
1291                     if (now - then) <= timeperiod:
1292                         tbl_dict[tst_name][u"data"][build] = (
1293                             tst_data[u"status"],
1294                             generated,
1295                             input_data.metadata(job, build).get(u"version",
1296                                                                 u""),
1297                             build
1298                         )
1299                 except (TypeError, KeyError) as err:
1300                     logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
1301
1302     max_fails = 0
1303     tbl_lst = list()
1304     for tst_data in tbl_dict.values():
1305         fails_nr = 0
1306         fails_last_date = u""
1307         fails_last_vpp = u""
1308         fails_last_csit = u""
1309         for val in tst_data[u"data"].values():
1310             if val[0] == u"FAIL":
1311                 fails_nr += 1
1312                 fails_last_date = val[1]
1313                 fails_last_vpp = val[2]
1314                 fails_last_csit = val[3]
1315         if fails_nr:
1316             max_fails = fails_nr if fails_nr > max_fails else max_fails
1317             tbl_lst.append(
1318                 [
1319                     tst_data[u"name"],
1320                     fails_nr,
1321                     fails_last_date,
1322                     fails_last_vpp,
1323                     f"mrr-daily-build-{fails_last_csit}"
1324                 ]
1325             )
1326
1327     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1328     tbl_sorted = list()
1329     for nrf in range(max_fails, -1, -1):
1330         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1331         tbl_sorted.extend(tbl_fails)
1332
1333     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1334     logging.info(f"    Writing file: {file_name}")
1335     with open(file_name, u"wt") as file_handler:
1336         file_handler.write(u",".join(header) + u"\n")
1337         for test in tbl_sorted:
1338             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1339
1340     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1341     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1342
1343
1344 def table_failed_tests_html(table, input_data):
1345     """Generate the table(s) with algorithm: table_failed_tests_html
1346     specified in the specification file.
1347
1348     :param table: Table to generate.
1349     :param input_data: Data to process.
1350     :type table: pandas.Series
1351     :type input_data: InputData
1352     """
1353
1354     _ = input_data
1355
1356     if not table.get(u"testbed", None):
1357         logging.error(
1358             f"The testbed is not defined for the table "
1359             f"{table.get(u'title', u'')}."
1360         )
1361         return
1362
1363     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1364
1365     try:
1366         with open(table[u"input-file"], u'rt') as csv_file:
1367             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1368     except KeyError:
1369         logging.warning(u"The input file is not defined.")
1370         return
1371     except csv.Error as err:
1372         logging.warning(
1373             f"Not possible to process the file {table[u'input-file']}.\n"
1374             f"{repr(err)}"
1375         )
1376         return
1377
1378     # Table:
1379     failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1380
1381     # Table header:
1382     trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1383     for idx, item in enumerate(csv_lst[0]):
1384         alignment = u"left" if idx == 0 else u"center"
1385         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1386         thead.text = item
1387
1388     # Rows:
1389     colors = (u"#e9f1fb", u"#d4e4f7")
1390     for r_idx, row in enumerate(csv_lst[1:]):
1391         background = colors[r_idx % 2]
1392         trow = ET.SubElement(
1393             failed_tests, u"tr", attrib=dict(bgcolor=background)
1394         )
1395
1396         # Columns:
1397         for c_idx, item in enumerate(row):
1398             tdata = ET.SubElement(
1399                 trow,
1400                 u"td",
1401                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1402             )
1403             # Name:
1404             if c_idx == 0:
1405                 ref = ET.SubElement(
1406                     tdata,
1407                     u"a",
1408                     attrib=dict(
1409                         href=f"../trending/"
1410                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1411                     )
1412                 )
1413                 ref.text = item
1414             else:
1415                 tdata.text = item
1416     try:
1417         with open(table[u"output-file"], u'w') as html_file:
1418             logging.info(f"    Writing file: {table[u'output-file']}")
1419             html_file.write(u".. raw:: html\n\n\t")
1420             html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
1421             html_file.write(u"\n\t<p><br><br></p>\n")
1422     except KeyError:
1423         logging.warning(u"The output file is not defined.")
1424         return
1425
1426
1427 def table_comparison(table, input_data):
1428     """Generate the table(s) with algorithm: table_comparison
1429     specified in the specification file.
1430
1431     :param table: Table to generate.
1432     :param input_data: Data to process.
1433     :type table: pandas.Series
1434     :type input_data: InputData
1435     """
1436     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1437
1438     # Transform the data
1439     logging.info(
1440         f"    Creating the data set for the {table.get(u'type', u'')} "
1441         f"{table.get(u'title', u'')}."
1442     )
1443
1444     columns = table.get(u"columns", None)
1445     if not columns:
1446         logging.error(
1447             f"No columns specified for {table.get(u'title', u'')}. Skipping."
1448         )
1449         return
1450
1451     cols = list()
1452     for idx, col in enumerate(columns):
1453         if col.get(u"data-set", None) is None:
1454             logging.warning(f"No data for column {col.get(u'title', u'')}")
1455             continue
1456         tag = col.get(u"tag", None)
1457         data = input_data.filter_data(
1458             table,
1459             params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1460             data=col[u"data-set"],
1461             continue_on_error=True
1462         )
1463         col_data = {
1464             u"title": col.get(u"title", f"Column{idx}"),
1465             u"data": dict()
1466         }
1467         for builds in data.values:
1468             for build in builds:
1469                 for tst_name, tst_data in build.items():
1470                     if tag and tag not in tst_data[u"tags"]:
1471                         continue
1472                     tst_name_mod = \
1473                         _tpc_modify_test_name(tst_name, ignore_nic=True).\
1474                         replace(u"2n1l-", u"")
1475                     if col_data[u"data"].get(tst_name_mod, None) is None:
1476                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
1477                         if u"across testbeds" in table[u"title"].lower() or \
1478                                 u"across topologies" in table[u"title"].lower():
1479                             name = _tpc_modify_displayed_test_name(name)
1480                         col_data[u"data"][tst_name_mod] = {
1481                             u"name": name,
1482                             u"replace": True,
1483                             u"data": list(),
1484                             u"mean": None,
1485                             u"stdev": None
1486                         }
1487                     _tpc_insert_data(
1488                         target=col_data[u"data"][tst_name_mod],
1489                         src=tst_data,
1490                         include_tests=table[u"include-tests"]
1491                     )
1492
1493         replacement = col.get(u"data-replacement", None)
1494         if replacement:
1495             rpl_data = input_data.filter_data(
1496                 table,
1497                 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1498                 data=replacement,
1499                 continue_on_error=True
1500             )
1501             for builds in rpl_data.values:
1502                 for build in builds:
1503                     for tst_name, tst_data in build.items():
1504                         if tag and tag not in tst_data[u"tags"]:
1505                             continue
1506                         tst_name_mod = \
1507                             _tpc_modify_test_name(tst_name, ignore_nic=True).\
1508                             replace(u"2n1l-", u"")
1509                         if col_data[u"data"].get(tst_name_mod, None) is None:
1510                             name = tst_data[u'name'].rsplit(u'-', 1)[0]
1511                             if u"across testbeds" in table[u"title"].lower() \
1512                                     or u"across topologies" in \
1513                                     table[u"title"].lower():
1514                                 name = _tpc_modify_displayed_test_name(name)
1515                             col_data[u"data"][tst_name_mod] = {
1516                                 u"name": name,
1517                                 u"replace": False,
1518                                 u"data": list(),
1519                                 u"mean": None,
1520                                 u"stdev": None
1521                             }
1522                         if col_data[u"data"][tst_name_mod][u"replace"]:
1523                             col_data[u"data"][tst_name_mod][u"replace"] = False
1524                             col_data[u"data"][tst_name_mod][u"data"] = list()
1525                         _tpc_insert_data(
1526                             target=col_data[u"data"][tst_name_mod],
1527                             src=tst_data,
1528                             include_tests=table[u"include-tests"]
1529                         )
1530
1531         if table[u"include-tests"] in (u"NDR", u"PDR"):
1532             for tst_name, tst_data in col_data[u"data"].items():
1533                 if tst_data[u"data"]:
1534                     tst_data[u"mean"] = mean(tst_data[u"data"])
1535                     tst_data[u"stdev"] = stdev(tst_data[u"data"])
1536
1537         cols.append(col_data)
1538
1539     tbl_dict = dict()
1540     for col in cols:
1541         for tst_name, tst_data in col[u"data"].items():
1542             if tbl_dict.get(tst_name, None) is None:
1543                 tbl_dict[tst_name] = {
1544                     "name": tst_data[u"name"]
1545                 }
1546             tbl_dict[tst_name][col[u"title"]] = {
1547                 u"mean": tst_data[u"mean"],
1548                 u"stdev": tst_data[u"stdev"]
1549             }
1550
1551     if not tbl_dict:
1552         logging.warning(f"No data for table {table.get(u'title', u'')}!")
1553         return
1554
1555     tbl_lst = list()
1556     for tst_data in tbl_dict.values():
1557         row = [tst_data[u"name"], ]
1558         for col in cols:
1559             row.append(tst_data.get(col[u"title"], None))
1560         tbl_lst.append(row)
1561
1562     comparisons = table.get(u"comparisons", None)
1563     if comparisons and isinstance(comparisons, list):
1564         for idx, comp in enumerate(comparisons):
1565             try:
1566                 col_ref = int(comp[u"reference"])
1567                 col_cmp = int(comp[u"compare"])
1568             except KeyError:
1569                 logging.warning(u"Comparison: No references defined! Skipping.")
1570                 comparisons.pop(idx)
1571                 continue
1572             if not (0 < col_ref <= len(cols) and
1573                     0 < col_cmp <= len(cols)) or \
1574                     col_ref == col_cmp:
1575                 logging.warning(f"Wrong values of reference={col_ref} "
1576                                 f"and/or compare={col_cmp}. Skipping.")
1577                 comparisons.pop(idx)
1578                 continue
1579
1580     tbl_cmp_lst = list()
1581     if comparisons:
1582         for row in tbl_lst:
1583             new_row = deepcopy(row)
1584             add_to_tbl = False
1585             for comp in comparisons:
1586                 ref_itm = row[int(comp[u"reference"])]
1587                 if ref_itm is None and \
1588                         comp.get(u"reference-alt", None) is not None:
1589                     ref_itm = row[int(comp[u"reference-alt"])]
1590                 cmp_itm = row[int(comp[u"compare"])]
1591                 if ref_itm is not None and cmp_itm is not None and \
1592                         ref_itm[u"mean"] is not None and \
1593                         cmp_itm[u"mean"] is not None and \
1594                         ref_itm[u"stdev"] is not None and \
1595                         cmp_itm[u"stdev"] is not None:
1596                     delta, d_stdev = relative_change_stdev(
1597                         ref_itm[u"mean"], cmp_itm[u"mean"],
1598                         ref_itm[u"stdev"], cmp_itm[u"stdev"]
1599                     )
1600                     new_row.append(
1601                         {
1602                             u"mean": delta * 1e6,
1603                             u"stdev": d_stdev * 1e6
1604                         }
1605                     )
1606                     add_to_tbl = True
1607                 else:
1608                     new_row.append(None)
1609             if add_to_tbl:
1610                 tbl_cmp_lst.append(new_row)
1611
1612     tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
1613     tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
1614
1615     rcas = list()
1616     rca_in = table.get(u"rca", None)
1617     if rca_in and isinstance(rca_in, list):
1618         for idx, itm in enumerate(rca_in):
1619             try:
1620                 with open(itm.get(u"data", u""), u"r") as rca_file:
1621                     rcas.append(
1622                         {
1623                             u"title": itm.get(u"title", f"RCA{idx}"),
1624                             u"data": load(rca_file, Loader=FullLoader)
1625                         }
1626                     )
1627             except (YAMLError, IOError) as err:
1628                 logging.warning(
1629                     f"The RCA file {itm.get(u'data', u'')} does not exist or "
1630                     f"it is corrupted!"
1631                 )
1632                 logging.debug(repr(err))
1633
1634     tbl_for_csv = list()
1635     for line in tbl_cmp_lst:
1636         row = [line[0], ]
1637         for idx, itm in enumerate(line[1:]):
1638             if itm is None:
1639                 row.append(u"NT")
1640                 row.append(u"NT")
1641             else:
1642                 row.append(round(float(itm[u'mean']) / 1e6, 3))
1643                 row.append(round(float(itm[u'stdev']) / 1e6, 3))
1644         for rca in rcas:
1645             rca_nr = rca[u"data"].get(row[0], u"-")
1646             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1647         tbl_for_csv.append(row)
1648
1649     header_csv = [u"Test Case", ]
1650     for col in cols:
1651         header_csv.append(f"Avg({col[u'title']})")
1652         header_csv.append(f"Stdev({col[u'title']})")
1653     for comp in comparisons:
1654         header_csv.append(
1655             f"Avg({comp.get(u'title', u'')})"
1656         )
1657         header_csv.append(
1658             f"Stdev({comp.get(u'title', u'')})"
1659         )
1660     header_csv.extend([rca[u"title"] for rca in rcas])
1661
1662     legend_lst = table.get(u"legend", None)
1663     if legend_lst is None:
1664         legend = u""
1665     else:
1666         legend = u"\n" + u"\n".join(legend_lst) + u"\n"
1667
1668     footnote = u""
1669     for rca in rcas:
1670         footnote += f"\n{rca[u'title']}:\n"
1671         footnote += rca[u"data"].get(u"footnote", u"")
1672
1673     csv_file = f"{table[u'output-file']}-csv.csv"
1674     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1675         file_handler.write(
1676             u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
1677         )
1678         for test in tbl_for_csv:
1679             file_handler.write(
1680                 u",".join([f'"{item}"' for item in test]) + u"\n"
1681             )
1682         if legend_lst:
1683             for item in legend_lst:
1684                 file_handler.write(f'"{item}"\n')
1685         if footnote:
1686             for itm in footnote.split(u"\n"):
1687                 file_handler.write(f'"{itm}"\n')
1688
1689     tbl_tmp = list()
1690     max_lens = [0, ] * len(tbl_cmp_lst[0])
1691     for line in tbl_cmp_lst:
1692         row = [line[0], ]
1693         for idx, itm in enumerate(line[1:]):
1694             if itm is None:
1695                 new_itm = u"NT"
1696             else:
1697                 if idx < len(cols):
1698                     new_itm = (
1699                         f"{round(float(itm[u'mean']) / 1e6, 1)} "
1700                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1701                         replace(u"nan", u"NaN")
1702                     )
1703                 else:
1704                     new_itm = (
1705                         f"{round(float(itm[u'mean']) / 1e6, 1):+} "
1706                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1707                         replace(u"nan", u"NaN")
1708                     )
1709             if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]:
1710                 max_lens[idx] = len(new_itm.rsplit(u" ", 1)[-1])
1711             row.append(new_itm)
1712
1713         tbl_tmp.append(row)
1714
1715     tbl_final = list()
1716     for line in tbl_tmp:
1717         row = [line[0], ]
1718         for idx, itm in enumerate(line[1:]):
1719             if itm in (u"NT", u"NaN"):
1720                 row.append(itm)
1721                 continue
1722             itm_lst = itm.rsplit(u"\u00B1", 1)
1723             itm_lst[-1] = \
1724                 f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
1725             row.append(u"\u00B1".join(itm_lst))
1726         for rca in rcas:
1727             rca_nr = rca[u"data"].get(row[0], u"-")
1728             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1729
1730         tbl_final.append(row)
1731
1732     header = [u"Test Case", ]
1733     header.extend([col[u"title"] for col in cols])
1734     header.extend([comp.get(u"title", u"") for comp in comparisons])
1735     header.extend([rca[u"title"] for rca in rcas])
1736
1737     # Generate csv tables:
1738     csv_file = f"{table[u'output-file']}.csv"
1739     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1740         file_handler.write(u";".join(header) + u"\n")
1741         for test in tbl_final:
1742             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1743
1744     # Generate txt table:
1745     txt_file_name = f"{table[u'output-file']}.txt"
1746     convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
1747
1748     with open(txt_file_name, u'a', encoding='utf-8') as txt_file:
1749         txt_file.write(legend)
1750         txt_file.write(footnote)
1751         if legend or footnote:
1752             txt_file.write(u"\n:END")
1753
1754     # Generate html table:
1755     _tpc_generate_html_table(
1756         header,
1757         tbl_final,
1758         table[u'output-file'],
1759         legend=legend,
1760         footnote=footnote,
1761         sort_data=False,
1762         title=table.get(u"title", u"")
1763     )
1764
1765
1766 def table_weekly_comparison(table, in_data):
1767     """Generate the table(s) with algorithm: table_weekly_comparison
1768     specified in the specification file.
1769
1770     :param table: Table to generate.
1771     :param in_data: Data to process.
1772     :type table: pandas.Series
1773     :type in_data: InputData
1774     """
1775     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1776
1777     # Transform the data
1778     logging.info(
1779         f"    Creating the data set for the {table.get(u'type', u'')} "
1780         f"{table.get(u'title', u'')}."
1781     )
1782
1783     incl_tests = table.get(u"include-tests", None)
1784     if incl_tests not in (u"NDR", u"PDR"):
1785         logging.error(f"Wrong tests to include specified ({incl_tests}).")
1786         return
1787
1788     nr_cols = table.get(u"nr-of-data-columns", None)
1789     if not nr_cols or nr_cols < 2:
1790         logging.error(
1791             f"No columns specified for {table.get(u'title', u'')}. Skipping."
1792         )
1793         return
1794
1795     data = in_data.filter_data(
1796         table,
1797         params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1798         continue_on_error=True
1799     )
1800
1801     header = [
1802         [u"VPP Version", ],
1803         [u"Start Timestamp", ],
1804         [u"CSIT Build", ],
1805         [u"CSIT Testbed", ]
1806     ]
1807     tbl_dict = dict()
1808     idx = 0
1809     tb_tbl = table.get(u"testbeds", None)
1810     for job_name, job_data in data.items():
1811         for build_nr, build in job_data.items():
1812             if idx >= nr_cols:
1813                 break
1814             if build.empty:
1815                 continue
1816
1817             tb_ip = in_data.metadata(job_name, build_nr).get(u"testbed", u"")
1818             if tb_ip and tb_tbl:
1819                 testbed = tb_tbl.get(tb_ip, u"")
1820             else:
1821                 testbed = u""
1822             header[2].insert(1, build_nr)
1823             header[3].insert(1, testbed)
1824             header[1].insert(
1825                 1, in_data.metadata(job_name, build_nr).get(u"generated", u"")
1826             )
1827             header[0].insert(
1828                 1, in_data.metadata(job_name, build_nr).get(u"version", u"")
1829             )
1830
1831             for tst_name, tst_data in build.items():
1832                 tst_name_mod = \
1833                     _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
1834                 if not tbl_dict.get(tst_name_mod, None):
1835                     tbl_dict[tst_name_mod] = dict(
1836                         name=tst_data[u'name'].rsplit(u'-', 1)[0],
1837                     )
1838                 try:
1839                     tbl_dict[tst_name_mod][-idx - 1] = \
1840                         tst_data[u"throughput"][incl_tests][u"LOWER"]
1841                 except (TypeError, IndexError, KeyError, ValueError):
1842                     pass
1843             idx += 1
1844
1845     if idx < nr_cols:
1846         logging.error(u"Not enough data to build the table! Skipping")
1847         return
1848
1849     cmp_dict = dict()
1850     for idx, cmp in enumerate(table.get(u"comparisons", list())):
1851         idx_ref = cmp.get(u"reference", None)
1852         idx_cmp = cmp.get(u"compare", None)
1853         if idx_ref is None or idx_cmp is None:
1854             continue
1855         header[0].append(
1856             f"Diff({header[0][idx_ref - idx].split(u'~')[-1]} vs "
1857             f"{header[0][idx_cmp - idx].split(u'~')[-1]})"
1858         )
1859         header[1].append(u"")
1860         header[2].append(u"")
1861         header[3].append(u"")
1862         for tst_name, tst_data in tbl_dict.items():
1863             if not cmp_dict.get(tst_name, None):
1864                 cmp_dict[tst_name] = list()
1865             ref_data = tst_data.get(idx_ref, None)
1866             cmp_data = tst_data.get(idx_cmp, None)
1867             if ref_data is None or cmp_data is None:
1868                 cmp_dict[tst_name].append(float('nan'))
1869             else:
1870                 cmp_dict[tst_name].append(
1871                     relative_change(ref_data, cmp_data)
1872                 )
1873
1874     tbl_lst = list()
1875     for tst_name, tst_data in tbl_dict.items():
1876         itm_lst = [tst_data[u"name"], ]
1877         for idx in range(nr_cols):
1878             item = tst_data.get(-idx - 1, None)
1879             if item is None:
1880                 itm_lst.insert(1, None)
1881             else:
1882                 itm_lst.insert(1, round(item / 1e6, 1))
1883         itm_lst.extend(
1884             [
1885                 None if itm is None else round(itm, 1)
1886                 for itm in cmp_dict[tst_name]
1887             ]
1888         )
1889         tbl_lst.append(itm_lst)
1890
1891     tbl_lst.sort(key=lambda rel: rel[0], reverse=False)
1892     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1893
1894     # Generate csv table:
1895     csv_file = f"{table[u'output-file']}.csv"
1896     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1897         for hdr in header:
1898             file_handler.write(u",".join(hdr) + u"\n")
1899         for test in tbl_lst:
1900             file_handler.write(u",".join(
1901                 [
1902                     str(item).replace(u"None", u"-").replace(u"nan", u"-").
1903                     replace(u"null", u"-") for item in test
1904                 ]
1905             ) + u"\n")
1906
1907     txt_file = f"{table[u'output-file']}.txt"
1908     convert_csv_to_pretty_txt(csv_file, txt_file, delimiter=u",")
1909
1910     # Reorganize header in txt table
1911     txt_table = list()
1912     with open(txt_file, u"rt", encoding='utf-8') as file_handler:
1913         for line in file_handler:
1914             txt_table.append(line)
1915     try:
1916         txt_table.insert(5, txt_table.pop(2))
1917         with open(txt_file, u"wt", encoding='utf-8') as file_handler:
1918             file_handler.writelines(txt_table)
1919     except IndexError:
1920         pass
1921
1922     # Generate html table:
1923     hdr_html = [
1924         u"<br>".join(row) for row in zip(*header)
1925     ]
1926     _tpc_generate_html_table(
1927         hdr_html,
1928         tbl_lst,
1929         table[u'output-file'],
1930         sort_data=True,
1931         title=table.get(u"title", u""),
1932         generate_rst=False
1933     )