resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import re
  21
  22 from collections import OrderedDict
  23 from xml.etree import ElementTree as ET
  24 from datetime import datetime as dt
  25 from datetime import timedelta
  26 from copy import deepcopy
  27
  28 import plotly.graph_objects as go
  29 import plotly.offline as ploff
  30 import pandas as pd
  31
  32 from numpy import nan, isnan
  33 from yaml import load, FullLoader, YAMLError
  34
  35 from pal_utils import mean, stdev, classify_anomalies, \
  36     convert_csv_to_pretty_txt, relative_change_stdev, relative_change
  37
  38
  39 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
  40
  41
  42 def generate_tables(spec, data):
  43     """Generate all tables specified in the specification file.
  44
  45     :param spec: Specification read from the specification file.
  46     :param data: Data to process.
  47     :type spec: Specification
  48     :type data: InputData
  49     """
  50
  51     generator = {
  52         u"table_merged_details": table_merged_details,
  53         u"table_soak_vs_ndr": table_soak_vs_ndr,
  54         u"table_perf_trending_dash": table_perf_trending_dash,
  55         u"table_perf_trending_dash_html": table_perf_trending_dash_html,
  56         u"table_last_failed_tests": table_last_failed_tests,
  57         u"table_failed_tests": table_failed_tests,
  58         u"table_failed_tests_html": table_failed_tests_html,
  59         u"table_oper_data_html": table_oper_data_html,
  60         u"table_comparison": table_comparison,
  61         u"table_weekly_comparison": table_weekly_comparison
  62     }
  63
  64     logging.info(u"Generating the tables ...")
  65     for table in spec.tables:
  66         try:
  67             if table[u"algorithm"] == u"table_weekly_comparison":
  68                 table[u"testbeds"] = spec.environment.get(u"testbeds", None)
  69             generator[table[u"algorithm"]](table, data)
  70         except NameError as err:
  71             logging.error(
  72                 f"Probably algorithm {table[u'algorithm']} is not defined: "
  73                 f"{repr(err)}"
  74             )
  75     logging.info(u"Done.")
  76
  77
  78 def table_oper_data_html(table, input_data):
  79     """Generate the table(s) with algorithm: html_table_oper_data
  80     specified in the specification file.
  81
  82     :param table: Table to generate.
  83     :param input_data: Data to process.
  84     :type table: pandas.Series
  85     :type input_data: InputData
  86     """
  87
  88     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
  89     # Transform the data
  90     logging.info(
  91         f"    Creating the data set for the {table.get(u'type', u'')} "
  92         f"{table.get(u'title', u'')}."
  93     )
  94     data = input_data.filter_data(
  95         table,
  96         params=[u"name", u"parent", u"show-run", u"type"],
  97         continue_on_error=True
  98     )
  99     if data.empty:
 100         return
 101     data = input_data.merge_data(data)
 102
 103     sort_tests = table.get(u"sort", None)
 104     if sort_tests:
 105         args = dict(
 106             inplace=True,
 107             ascending=(sort_tests == u"ascending")
 108         )
 109         data.sort_index(**args)
 110
 111     suites = input_data.filter_data(
 112         table,
 113         continue_on_error=True,
 114         data_set=u"suites"
 115     )
 116     if suites.empty:
 117         return
 118     suites = input_data.merge_data(suites)
 119
 120     def _generate_html_table(tst_data):
 121         """Generate an HTML table with operational data for the given test.
 122
 123         :param tst_data: Test data to be used to generate the table.
 124         :type tst_data: pandas.Series
 125         :returns: HTML table with operational data.
 126         :rtype: str
 127         """
 128
 129         colors = {
 130             u"header": u"#7eade7",
 131             u"empty": u"#ffffff",
 132             u"body": (u"#e9f1fb", u"#d4e4f7")
 133         }
 134
 135         tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
 136
 137         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
 138         thead = ET.SubElement(
 139             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 140         )
 141         thead.text = tst_data[u"name"]
 142
 143         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 144         thead = ET.SubElement(
 145             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 146         )
 147         thead.text = u"\t"
 148
 149         if tst_data.get(u"show-run", u"No Data") == u"No Data":
 150             trow = ET.SubElement(
 151                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 152             )
 153             tcol = ET.SubElement(
 154                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 155             )
 156             tcol.text = u"No Data"
 157
 158             trow = ET.SubElement(
 159                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 160             )
 161             thead = ET.SubElement(
 162                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 163             )
 164             font = ET.SubElement(
 165                 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 166             )
 167             font.text = u"."
 168             return str(ET.tostring(tbl, encoding=u"unicode"))
 169
 170         tbl_hdr = (
 171             u"Name",
 172             u"Nr of Vectors",
 173             u"Nr of Packets",
 174             u"Suspends",
 175             u"Cycles per Packet",
 176             u"Average Vector Size"
 177         )
 178
 179         for dut_data in tst_data[u"show-run"].values():
 180             trow = ET.SubElement(
 181                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 182             )
 183             tcol = ET.SubElement(
 184                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 185             )
 186             if dut_data.get(u"threads", None) is None:
 187                 tcol.text = u"No Data"
 188                 continue
 189
 190             bold = ET.SubElement(tcol, u"b")
 191             bold.text = (
 192                 f"Host IP: {dut_data.get(u'host', '')}, "
 193                 f"Socket: {dut_data.get(u'socket', '')}"
 194             )
 195             trow = ET.SubElement(
 196                 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 197             )
 198             thead = ET.SubElement(
 199                 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 200             )
 201             thead.text = u"\t"
 202
 203             for thread_nr, thread in dut_data[u"threads"].items():
 204                 trow = ET.SubElement(
 205                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 206                 )
 207                 tcol = ET.SubElement(
 208                     trow, u"td", attrib=dict(align=u"left", colspan=u"6")
 209                 )
 210                 bold = ET.SubElement(tcol, u"b")
 211                 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
 212                 trow = ET.SubElement(
 213                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
 214                 )
 215                 for idx, col in enumerate(tbl_hdr):
 216                     tcol = ET.SubElement(
 217                         trow, u"td",
 218                         attrib=dict(align=u"right" if idx else u"left")
 219                     )
 220                     font = ET.SubElement(
 221                         tcol, u"font", attrib=dict(size=u"2")
 222                     )
 223                     bold = ET.SubElement(font, u"b")
 224                     bold.text = col
 225                 for row_nr, row in enumerate(thread):
 226                     trow = ET.SubElement(
 227                         tbl, u"tr",
 228                         attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
 229                     )
 230                     for idx, col in enumerate(row):
 231                         tcol = ET.SubElement(
 232                             trow, u"td",
 233                             attrib=dict(align=u"right" if idx else u"left")
 234                         )
 235                         font = ET.SubElement(
 236                             tcol, u"font", attrib=dict(size=u"2")
 237                         )
 238                         if isinstance(col, float):
 239                             font.text = f"{col:.2f}"
 240                         else:
 241                             font.text = str(col)
 242                 trow = ET.SubElement(
 243                     tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
 244                 )
 245                 thead = ET.SubElement(
 246                     trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 247                 )
 248                 thead.text = u"\t"
 249
 250         trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
 251         thead = ET.SubElement(
 252             trow, u"th", attrib=dict(align=u"left", colspan=u"6")
 253         )
 254         font = ET.SubElement(
 255             thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
 256         )
 257         font.text = u"."
 258
 259         return str(ET.tostring(tbl, encoding=u"unicode"))
 260
 261     for suite in suites.values:
 262         html_table = str()
 263         for test_data in data.values:
 264             if test_data[u"parent"] not in suite[u"name"]:
 265                 continue
 266             html_table += _generate_html_table(test_data)
 267         if not html_table:
 268             continue
 269         try:
 270             file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
 271             with open(f"{file_name}", u'w') as html_file:
 272                 logging.info(f"    Writing file: {file_name}")
 273                 html_file.write(u".. raw:: html\n\n\t")
 274                 html_file.write(html_table)
 275                 html_file.write(u"\n\t<p><br><br></p>\n")
 276         except KeyError:
 277             logging.warning(u"The output file is not defined.")
 278             return
 279     logging.info(u"  Done.")
 280
 281
 282 def table_merged_details(table, input_data):
 283     """Generate the table(s) with algorithm: table_merged_details
 284     specified in the specification file.
 285
 286     :param table: Table to generate.
 287     :param input_data: Data to process.
 288     :type table: pandas.Series
 289     :type input_data: InputData
 290     """
 291
 292     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 293
 294     # Transform the data
 295     logging.info(
 296         f"    Creating the data set for the {table.get(u'type', u'')} "
 297         f"{table.get(u'title', u'')}."
 298     )
 299     data = input_data.filter_data(table, continue_on_error=True)
 300     data = input_data.merge_data(data)
 301
 302     sort_tests = table.get(u"sort", None)
 303     if sort_tests:
 304         args = dict(
 305             inplace=True,
 306             ascending=(sort_tests == u"ascending")
 307         )
 308         data.sort_index(**args)
 309
 310     suites = input_data.filter_data(
 311         table, continue_on_error=True, data_set=u"suites")
 312     suites = input_data.merge_data(suites)
 313
 314     # Prepare the header of the tables
 315     header = list()
 316     for column in table[u"columns"]:
 317         header.append(
 318             u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
 319         )
 320
 321     for suite in suites.values:
 322         # Generate data
 323         suite_name = suite[u"name"]
 324         table_lst = list()
 325         for test in data.keys():
 326             if data[test][u"parent"] not in suite_name:
 327                 continue
 328             row_lst = list()
 329             for column in table[u"columns"]:
 330                 try:
 331                     col_data = str(data[test][column[
 332                         u"data"].split(u" ")[1]]).replace(u'"', u'""')
 333                     # Do not include tests with "Test Failed" in test message
 334                     if u"Test Failed" in col_data:
 335                         continue
 336                     col_data = col_data.replace(
 337                         u"No Data", u"Not Captured     "
 338                     )
 339                     if column[u"data"].split(u" ")[1] in (u"name", ):
 340                         if len(col_data) > 30:
 341                             col_data_lst = col_data.split(u"-")
 342                             half = int(len(col_data_lst) / 2)
 343                             col_data = f"{u'-'.join(col_data_lst[:half])}" \
 344                                        f"- |br| " \
 345                                        f"{u'-'.join(col_data_lst[half:])}"
 346                         col_data = f" |prein| {col_data} |preout| "
 347                     elif column[u"data"].split(u" ")[1] in (u"msg", ):
 348                         # Temporary solution: remove NDR results from message:
 349                         if bool(table.get(u'remove-ndr', False)):
 350                             try:
 351                                 col_data = col_data.split(u" |br| ", 1)[1]
 352                             except IndexError:
 353                                 pass
 354                         col_data = f" |prein| {col_data} |preout| "
 355                     elif column[u"data"].split(u" ")[1] in \
 356                             (u"conf-history", u"show-run"):
 357                         col_data = col_data.replace(u" |br| ", u"", 1)
 358                         col_data = f" |prein| {col_data[:-5]} |preout| "
 359                     row_lst.append(f'"{col_data}"')
 360                 except KeyError:
 361                     row_lst.append(u'"Not captured"')
 362             if len(row_lst) == len(table[u"columns"]):
 363                 table_lst.append(row_lst)
 364
 365         # Write the data to file
 366         if table_lst:
 367             separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
 368             file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
 369             logging.info(f"      Writing file: {file_name}")
 370             with open(file_name, u"wt") as file_handler:
 371                 file_handler.write(u",".join(header) + u"\n")
 372                 for item in table_lst:
 373                     file_handler.write(u",".join(item) + u"\n")
 374
 375     logging.info(u"  Done.")
 376
 377
 378 def _tpc_modify_test_name(test_name, ignore_nic=False):
 379     """Modify a test name by replacing its parts.
 380
 381     :param test_name: Test name to be modified.
 382     :param ignore_nic: If True, NIC is removed from TC name.
 383     :type test_name: str
 384     :type ignore_nic: bool
 385     :returns: Modified test name.
 386     :rtype: str
 387     """
 388     test_name_mod = test_name.\
 389         replace(u"-ndrpdrdisc", u""). \
 390         replace(u"-ndrpdr", u"").\
 391         replace(u"-pdrdisc", u""). \
 392         replace(u"-ndrdisc", u"").\
 393         replace(u"-pdr", u""). \
 394         replace(u"-ndr", u""). \
 395         replace(u"1t1c", u"1c").\
 396         replace(u"2t1c", u"1c"). \
 397         replace(u"2t2c", u"2c").\
 398         replace(u"4t2c", u"2c"). \
 399         replace(u"4t4c", u"4c").\
 400         replace(u"8t4c", u"4c")
 401
 402     if ignore_nic:
 403         return re.sub(REGEX_NIC, u"", test_name_mod)
 404     return test_name_mod
 405
 406
 407 def _tpc_modify_displayed_test_name(test_name):
 408     """Modify a test name which is displayed in a table by replacing its parts.
 409
 410     :param test_name: Test name to be modified.
 411     :type test_name: str
 412     :returns: Modified test name.
 413     :rtype: str
 414     """
 415     return test_name.\
 416         replace(u"1t1c", u"1c").\
 417         replace(u"2t1c", u"1c"). \
 418         replace(u"2t2c", u"2c").\
 419         replace(u"4t2c", u"2c"). \
 420         replace(u"4t4c", u"4c").\
 421         replace(u"8t4c", u"4c")
 422
 423
 424 def _tpc_insert_data(target, src, include_tests):
 425     """Insert src data to the target structure.
 426
 427     :param target: Target structure where the data is placed.
 428     :param src: Source data to be placed into the target stucture.
 429     :param include_tests: Which results will be included (MRR, NDR, PDR).
 430     :type target: list
 431     :type src: dict
 432     :type include_tests: str
 433     """
 434     try:
 435         if include_tests == u"MRR":
 436             target[u"mean"] = src[u"result"][u"receive-rate"]
 437             target[u"stdev"] = src[u"result"][u"receive-stdev"]
 438         elif include_tests == u"PDR":
 439             target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
 440         elif include_tests == u"NDR":
 441             target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
 442     except (KeyError, TypeError):
 443         pass
 444
 445
 446 def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
 447                              footnote=u"", sort_data=True, title=u"",
 448                              generate_rst=True):
 449     """Generate html table from input data with simple sorting possibility.
 450
 451     :param header: Table header.
 452     :param data: Input data to be included in the table. It is a list of lists.
 453         Inner lists are rows in the table. All inner lists must be of the same
 454         length. The length of these lists must be the same as the length of the
 455         header.
 456     :param out_file_name: The name (relative or full path) where the
 457         generated html table is written.
 458     :param legend: The legend to display below the table.
 459     :param footnote: The footnote to display below the table (and legend).
 460     :param sort_data: If True the data sorting is enabled.
 461     :param title: The table (and file) title.
 462     :param generate_rst: If True, wrapping rst file is generated.
 463     :type header: list
 464     :type data: list of lists
 465     :type out_file_name: str
 466     :type legend: str
 467     :type footnote: str
 468     :type sort_data: bool
 469     :type title: str
 470     :type generate_rst: bool
 471     """
 472
 473     try:
 474         idx = header.index(u"Test Case")
 475     except ValueError:
 476         idx = 0
 477     params = {
 478         u"align-hdr": (
 479             [u"left", u"right"],
 480             [u"left", u"left", u"right"],
 481             [u"left", u"left", u"left", u"right"]
 482         ),
 483         u"align-itm": (
 484             [u"left", u"right"],
 485             [u"left", u"left", u"right"],
 486             [u"left", u"left", u"left", u"right"]
 487         ),
 488         u"width": ([15, 9], [4, 24, 10], [4, 4, 32, 10])
 489     }
 490
 491     df_data = pd.DataFrame(data, columns=header)
 492
 493     if sort_data:
 494         df_sorted = [df_data.sort_values(
 495             by=[key, header[idx]], ascending=[True, True]
 496             if key != header[idx] else [False, True]) for key in header]
 497         df_sorted_rev = [df_data.sort_values(
 498             by=[key, header[idx]], ascending=[False, True]
 499             if key != header[idx] else [True, True]) for key in header]
 500         df_sorted.extend(df_sorted_rev)
 501     else:
 502         df_sorted = df_data
 503
 504     fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
 505                    for idx in range(len(df_data))]]
 506     table_header = dict(
 507         values=[f"<b>{item.replace(u',', u',<br>')}</b>" for item in header],
 508         fill_color=u"#7eade7",
 509         align=params[u"align-hdr"][idx],
 510         font=dict(
 511             family=u"Courier New",
 512             size=12
 513         )
 514     )
 515
 516     fig = go.Figure()
 517
 518     if sort_data:
 519         for table in df_sorted:
 520             columns = [table.get(col) for col in header]
 521             fig.add_trace(
 522                 go.Table(
 523                     columnwidth=params[u"width"][idx],
 524                     header=table_header,
 525                     cells=dict(
 526                         values=columns,
 527                         fill_color=fill_color,
 528                         align=params[u"align-itm"][idx],
 529                         font=dict(
 530                             family=u"Courier New",
 531                             size=12
 532                         )
 533                     )
 534                 )
 535             )
 536
 537         buttons = list()
 538         menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
 539         menu_items.extend([f"<b>{itm}</b> (descending)" for itm in header])
 540         for idx, hdr in enumerate(menu_items):
 541             visible = [False, ] * len(menu_items)
 542             visible[idx] = True
 543             buttons.append(
 544                 dict(
 545                     label=hdr.replace(u" [Mpps]", u""),
 546                     method=u"update",
 547                     args=[{u"visible": visible}],
 548                 )
 549             )
 550
 551         fig.update_layout(
 552             updatemenus=[
 553                 go.layout.Updatemenu(
 554                     type=u"dropdown",
 555                     direction=u"down",
 556                     x=0.0,
 557                     xanchor=u"left",
 558                     y=1.002,
 559                     yanchor=u"bottom",
 560                     active=len(menu_items) - 1,
 561                     buttons=list(buttons)
 562                 )
 563             ],
 564         )
 565     else:
 566         fig.add_trace(
 567             go.Table(
 568                 columnwidth=params[u"width"][idx],
 569                 header=table_header,
 570                 cells=dict(
 571                     values=[df_sorted.get(col) for col in header],
 572                     fill_color=fill_color,
 573                     align=params[u"align-itm"][idx],
 574                     font=dict(
 575                         family=u"Courier New",
 576                         size=12
 577                     )
 578                 )
 579             )
 580         )
 581
 582     ploff.plot(
 583         fig,
 584         show_link=False,
 585         auto_open=False,
 586         filename=f"{out_file_name}_in.html"
 587     )
 588
 589     if not generate_rst:
 590         return
 591
 592     file_name = out_file_name.split(u"/")[-1]
 593     if u"vpp" in out_file_name:
 594         path = u"_tmp/src/vpp_performance_tests/comparisons/"
 595     else:
 596         path = u"_tmp/src/dpdk_performance_tests/comparisons/"
 597     with open(f"{path}{file_name}.rst", u"wt") as rst_file:
 598         rst_file.write(
 599             u"\n"
 600             u".. |br| raw:: html\n\n    <br />\n\n\n"
 601             u".. |prein| raw:: html\n\n    <pre>\n\n\n"
 602             u".. |preout| raw:: html\n\n    </pre>\n\n"
 603         )
 604         if title:
 605             rst_file.write(f"{title}\n")
 606             rst_file.write(f"{u'`' * len(title)}\n\n")
 607         rst_file.write(
 608             u".. raw:: html\n\n"
 609             f'    <iframe frameborder="0" scrolling="no" '
 610             f'width="1600" height="1200" '
 611             f'src="../..{out_file_name.replace(u"_build", u"")}_in.html">'
 612             f'</iframe>\n\n'
 613         )
 614         if legend:
 615             rst_file.write(legend[1:].replace(u"\n", u" |br| "))
 616         if footnote:
 617             rst_file.write(footnote.replace(u"\n", u" |br| ")[1:])
 618
 619
 620 def table_soak_vs_ndr(table, input_data):
 621     """Generate the table(s) with algorithm: table_soak_vs_ndr
 622     specified in the specification file.
 623
 624     :param table: Table to generate.
 625     :param input_data: Data to process.
 626     :type table: pandas.Series
 627     :type input_data: InputData
 628     """
 629
 630     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 631
 632     # Transform the data
 633     logging.info(
 634         f"    Creating the data set for the {table.get(u'type', u'')} "
 635         f"{table.get(u'title', u'')}."
 636     )
 637     data = input_data.filter_data(table, continue_on_error=True)
 638
 639     # Prepare the header of the table
 640     try:
 641         header = [
 642             u"Test Case",
 643             f"Avg({table[u'reference'][u'title']})",
 644             f"Stdev({table[u'reference'][u'title']})",
 645             f"Avg({table[u'compare'][u'title']})",
 646             f"Stdev{table[u'compare'][u'title']})",
 647             u"Diff",
 648             u"Stdev(Diff)"
 649         ]
 650         header_str = u";".join(header) + u"\n"
 651         legend = (
 652             u"\nLegend:\n"
 653             f"Avg({table[u'reference'][u'title']}): "
 654             f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
 655             f"from a series of runs of the listed tests.\n"
 656             f"Stdev({table[u'reference'][u'title']}): "
 657             f"Standard deviation value of {table[u'reference'][u'title']} "
 658             f"[Mpps] computed from a series of runs of the listed tests.\n"
 659             f"Avg({table[u'compare'][u'title']}): "
 660             f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
 661             f"a series of runs of the listed tests.\n"
 662             f"Stdev({table[u'compare'][u'title']}): "
 663             f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
 664             f"computed from a series of runs of the listed tests.\n"
 665             f"Diff({table[u'reference'][u'title']},"
 666             f"{table[u'compare'][u'title']}): "
 667             f"Percentage change calculated for mean values.\n"
 668             u"Stdev(Diff): "
 669             u"Standard deviation of percentage change calculated for mean "
 670             u"values.\n"
 671             u":END"
 672         )
 673     except (AttributeError, KeyError) as err:
 674         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
 675         return
 676
 677     # Create a list of available SOAK test results:
 678     tbl_dict = dict()
 679     for job, builds in table[u"compare"][u"data"].items():
 680         for build in builds:
 681             for tst_name, tst_data in data[job][str(build)].items():
 682                 if tst_data[u"type"] == u"SOAK":
 683                     tst_name_mod = tst_name.replace(u"-soak", u"")
 684                     if tbl_dict.get(tst_name_mod, None) is None:
 685                         groups = re.search(REGEX_NIC, tst_data[u"parent"])
 686                         nic = groups.group(0) if groups else u""
 687                         name = (
 688                             f"{nic}-"
 689                             f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
 690                         )
 691                         tbl_dict[tst_name_mod] = {
 692                             u"name": name,
 693                             u"ref-data": list(),
 694                             u"cmp-data": list()
 695                         }
 696                     try:
 697                         tbl_dict[tst_name_mod][u"cmp-data"].append(
 698                             tst_data[u"throughput"][u"LOWER"])
 699                     except (KeyError, TypeError):
 700                         pass
 701     tests_lst = tbl_dict.keys()
 702
 703     # Add corresponding NDR test results:
 704     for job, builds in table[u"reference"][u"data"].items():
 705         for build in builds:
 706             for tst_name, tst_data in data[job][str(build)].items():
 707                 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
 708                     replace(u"-mrr", u"")
 709                 if tst_name_mod not in tests_lst:
 710                     continue
 711                 try:
 712                     if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
 713                         continue
 714                     if table[u"include-tests"] == u"MRR":
 715                         result = (tst_data[u"result"][u"receive-rate"],
 716                                   tst_data[u"result"][u"receive-stdev"])
 717                     elif table[u"include-tests"] == u"PDR":
 718                         result = \
 719                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
 720                     elif table[u"include-tests"] == u"NDR":
 721                         result = \
 722                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
 723                     else:
 724                         result = None
 725                     if result is not None:
 726                         tbl_dict[tst_name_mod][u"ref-data"].append(
 727                             result)
 728                 except (KeyError, TypeError):
 729                     continue
 730
 731     tbl_lst = list()
 732     for tst_name in tbl_dict:
 733         item = [tbl_dict[tst_name][u"name"], ]
 734         data_r = tbl_dict[tst_name][u"ref-data"]
 735         if data_r:
 736             if table[u"include-tests"] == u"MRR":
 737                 data_r_mean = data_r[0][0]
 738                 data_r_stdev = data_r[0][1]
 739             else:
 740                 data_r_mean = mean(data_r)
 741                 data_r_stdev = stdev(data_r)
 742             item.append(round(data_r_mean / 1e6, 1))
 743             item.append(round(data_r_stdev / 1e6, 1))
 744         else:
 745             data_r_mean = None
 746             data_r_stdev = None
 747             item.extend([None, None])
 748         data_c = tbl_dict[tst_name][u"cmp-data"]
 749         if data_c:
 750             if table[u"include-tests"] == u"MRR":
 751                 data_c_mean = data_c[0][0]
 752                 data_c_stdev = data_c[0][1]
 753             else:
 754                 data_c_mean = mean(data_c)
 755                 data_c_stdev = stdev(data_c)
 756             item.append(round(data_c_mean / 1e6, 1))
 757             item.append(round(data_c_stdev / 1e6, 1))
 758         else:
 759             data_c_mean = None
 760             data_c_stdev = None
 761             item.extend([None, None])
 762         if data_r_mean is not None and data_c_mean is not None:
 763             delta, d_stdev = relative_change_stdev(
 764                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
 765             try:
 766                 item.append(round(delta))
 767             except ValueError:
 768                 item.append(delta)
 769             try:
 770                 item.append(round(d_stdev))
 771             except ValueError:
 772                 item.append(d_stdev)
 773             tbl_lst.append(item)
 774
 775     # Sort the table according to the relative change
 776     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 777
 778     # Generate csv tables:
 779     csv_file = f"{table[u'output-file']}.csv"
 780     with open(csv_file, u"wt") as file_handler:
 781         file_handler.write(header_str)
 782         for test in tbl_lst:
 783             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 784
 785     convert_csv_to_pretty_txt(
 786         csv_file, f"{table[u'output-file']}.txt", delimiter=u";"
 787     )
 788     with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
 789         txt_file.write(legend)
 790
 791     # Generate html table:
 792     _tpc_generate_html_table(
 793         header,
 794         tbl_lst,
 795         table[u'output-file'],
 796         legend=legend,
 797         title=table.get(u"title", u"")
 798     )
 799
 800
 801 def table_perf_trending_dash(table, input_data):
 802     """Generate the table(s) with algorithm:
 803     table_perf_trending_dash
 804     specified in the specification file.
 805
 806     :param table: Table to generate.
 807     :param input_data: Data to process.
 808     :type table: pandas.Series
 809     :type input_data: InputData
 810     """
 811
 812     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 813
 814     # Transform the data
 815     logging.info(
 816         f"    Creating the data set for the {table.get(u'type', u'')} "
 817         f"{table.get(u'title', u'')}."
 818     )
 819     data = input_data.filter_data(table, continue_on_error=True)
 820
 821     # Prepare the header of the tables
 822     header = [
 823         u"Test Case",
 824         u"Trend [Mpps]",
 825         u"Short-Term Change [%]",
 826         u"Long-Term Change [%]",
 827         u"Regressions [#]",
 828         u"Progressions [#]"
 829     ]
 830     header_str = u",".join(header) + u"\n"
 831
 832     incl_tests = table.get(u"include-tests", u"MRR")
 833
 834     # Prepare data to the table:
 835     tbl_dict = dict()
 836     for job, builds in table[u"data"].items():
 837         for build in builds:
 838             for tst_name, tst_data in data[job][str(build)].items():
 839                 if tst_name.lower() in table.get(u"ignore-list", list()):
 840                     continue
 841                 if tbl_dict.get(tst_name, None) is None:
 842                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
 843                     if not groups:
 844                         continue
 845                     nic = groups.group(0)
 846                     tbl_dict[tst_name] = {
 847                         u"name": f"{nic}-{tst_data[u'name']}",
 848                         u"data": OrderedDict()
 849                     }
 850                 try:
 851                     if incl_tests == u"MRR":
 852                         tbl_dict[tst_name][u"data"][str(build)] = \
 853                             tst_data[u"result"][u"receive-rate"]
 854                     elif incl_tests == u"NDR":
 855                         tbl_dict[tst_name][u"data"][str(build)] = \
 856                             tst_data[u"throughput"][u"NDR"][u"LOWER"]
 857                     elif incl_tests == u"PDR":
 858                         tbl_dict[tst_name][u"data"][str(build)] = \
 859                             tst_data[u"throughput"][u"PDR"][u"LOWER"]
 860                 except (TypeError, KeyError):
 861                     pass  # No data in output.xml for this test
 862
 863     tbl_lst = list()
 864     for tst_name in tbl_dict:
 865         data_t = tbl_dict[tst_name][u"data"]
 866         if len(data_t) < 2:
 867             continue
 868
 869         classification_lst, avgs, _ = classify_anomalies(data_t)
 870
 871         win_size = min(len(data_t), table[u"window"])
 872         long_win_size = min(len(data_t), table[u"long-trend-window"])
 873
 874         try:
 875             max_long_avg = max(
 876                 [x for x in avgs[-long_win_size:-win_size]
 877                  if not isnan(x)])
 878         except ValueError:
 879             max_long_avg = nan
 880         last_avg = avgs[-1]
 881         avg_week_ago = avgs[max(-win_size, -len(avgs))]
 882
 883         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
 884             rel_change_last = nan
 885         else:
 886             rel_change_last = round(
 887                 ((last_avg - avg_week_ago) / avg_week_ago) * 1e2, 2)
 888
 889         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
 890             rel_change_long = nan
 891         else:
 892             rel_change_long = round(
 893                 ((last_avg - max_long_avg) / max_long_avg) * 1e2, 2)
 894
 895         if classification_lst:
 896             if isnan(rel_change_last) and isnan(rel_change_long):
 897                 continue
 898             if isnan(last_avg) or isnan(rel_change_last) or \
 899                     isnan(rel_change_long):
 900                 continue
 901             tbl_lst.append(
 902                 [tbl_dict[tst_name][u"name"],
 903                  round(last_avg / 1e6, 2),
 904                  rel_change_last,
 905                  rel_change_long,
 906                  classification_lst[-win_size+1:].count(u"regression"),
 907                  classification_lst[-win_size+1:].count(u"progression")])
 908
 909     tbl_lst.sort(key=lambda rel: rel[0])
 910
 911     tbl_sorted = list()
 912     for nrr in range(table[u"window"], -1, -1):
 913         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
 914         for nrp in range(table[u"window"], -1, -1):
 915             tbl_out = [item for item in tbl_reg if item[5] == nrp]
 916             tbl_out.sort(key=lambda rel: rel[2])
 917             tbl_sorted.extend(tbl_out)
 918
 919     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
 920
 921     logging.info(f"    Writing file: {file_name}")
 922     with open(file_name, u"wt") as file_handler:
 923         file_handler.write(header_str)
 924         for test in tbl_sorted:
 925             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
 926
 927     logging.info(f"    Writing file: {table[u'output-file']}.txt")
 928     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
 929
 930
 931 def _generate_url(testbed, test_name):
 932     """Generate URL to a trending plot from the name of the test case.
 933
 934     :param testbed: The testbed used for testing.
 935     :param test_name: The name of the test case.
 936     :type testbed: str
 937     :type test_name: str
 938     :returns: The URL to the plot with the trending data for the given test
 939         case.
 940     :rtype str
 941     """
 942
 943     if u"x520" in test_name:
 944         nic = u"x520"
 945     elif u"x710" in test_name:
 946         nic = u"x710"
 947     elif u"xl710" in test_name:
 948         nic = u"xl710"
 949     elif u"xxv710" in test_name:
 950         nic = u"xxv710"
 951     elif u"vic1227" in test_name:
 952         nic = u"vic1227"
 953     elif u"vic1385" in test_name:
 954         nic = u"vic1385"
 955     elif u"x553" in test_name:
 956         nic = u"x553"
 957     elif u"cx556" in test_name or u"cx556a" in test_name:
 958         nic = u"cx556a"
 959     else:
 960         nic = u""
 961
 962     if u"64b" in test_name:
 963         frame_size = u"64b"
 964     elif u"78b" in test_name:
 965         frame_size = u"78b"
 966     elif u"imix" in test_name:
 967         frame_size = u"imix"
 968     elif u"9000b" in test_name:
 969         frame_size = u"9000b"
 970     elif u"1518b" in test_name:
 971         frame_size = u"1518b"
 972     elif u"114b" in test_name:
 973         frame_size = u"114b"
 974     else:
 975         frame_size = u""
 976
 977     if u"1t1c" in test_name or \
 978         (u"-1c-" in test_name and
 979          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 980         cores = u"1t1c"
 981     elif u"2t2c" in test_name or \
 982          (u"-2c-" in test_name and
 983           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 984         cores = u"2t2c"
 985     elif u"4t4c" in test_name or \
 986          (u"-4c-" in test_name and
 987           testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
 988         cores = u"4t4c"
 989     elif u"2t1c" in test_name or \
 990          (u"-1c-" in test_name and
 991           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
 992         cores = u"2t1c"
 993     elif u"4t2c" in test_name or \
 994          (u"-2c-" in test_name and
 995           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
 996         cores = u"4t2c"
 997     elif u"8t4c" in test_name or \
 998          (u"-4c-" in test_name and
 999           testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1000         cores = u"8t4c"
1001     else:
1002         cores = u""
1003
1004     if u"testpmd" in test_name:
1005         driver = u"testpmd"
1006     elif u"l3fwd" in test_name:
1007         driver = u"l3fwd"
1008     elif u"avf" in test_name:
1009         driver = u"avf"
1010     elif u"rdma" in test_name:
1011         driver = u"rdma"
1012     elif u"dnv" in testbed or u"tsh" in testbed:
1013         driver = u"ixgbe"
1014     else:
1015         driver = u"dpdk"
1016
1017     if u"acl" in test_name or \
1018             u"macip" in test_name or \
1019             u"nat" in test_name or \
1020             u"policer" in test_name or \
1021             u"cop" in test_name:
1022         bsf = u"features"
1023     elif u"scale" in test_name:
1024         bsf = u"scale"
1025     elif u"base" in test_name:
1026         bsf = u"base"
1027     else:
1028         bsf = u"base"
1029
1030     if u"114b" in test_name and u"vhost" in test_name:
1031         domain = u"vts"
1032     elif u"testpmd" in test_name or u"l3fwd" in test_name:
1033         domain = u"dpdk"
1034     elif u"memif" in test_name:
1035         domain = u"container_memif"
1036     elif u"srv6" in test_name:
1037         domain = u"srv6"
1038     elif u"vhost" in test_name:
1039         domain = u"vhost"
1040         if u"vppl2xc" in test_name:
1041             driver += u"-vpp"
1042         else:
1043             driver += u"-testpmd"
1044         if u"lbvpplacp" in test_name:
1045             bsf += u"-link-bonding"
1046     elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1047         domain = u"nf_service_density_vnfc"
1048     elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1049         domain = u"nf_service_density_cnfc"
1050     elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1051         domain = u"nf_service_density_cnfp"
1052     elif u"ipsec" in test_name:
1053         domain = u"ipsec"
1054         if u"sw" in test_name:
1055             bsf += u"-sw"
1056         elif u"hw" in test_name:
1057             bsf += u"-hw"
1058     elif u"ethip4vxlan" in test_name:
1059         domain = u"ip4_tunnels"
1060     elif u"ip4base" in test_name or u"ip4scale" in test_name:
1061         domain = u"ip4"
1062     elif u"ip6base" in test_name or u"ip6scale" in test_name:
1063         domain = u"ip6"
1064     elif u"l2xcbase" in test_name or \
1065             u"l2xcscale" in test_name or \
1066             u"l2bdbasemaclrn" in test_name or \
1067             u"l2bdscale" in test_name or \
1068             u"l2patch" in test_name:
1069         domain = u"l2"
1070     else:
1071         domain = u""
1072
1073     file_name = u"-".join((domain, testbed, nic)) + u".html#"
1074     anchor_name = u"-".join((frame_size, cores, bsf, driver))
1075
1076     return file_name + anchor_name
1077
1078
1079 def table_perf_trending_dash_html(table, input_data):
1080     """Generate the table(s) with algorithm:
1081     table_perf_trending_dash_html specified in the specification
1082     file.
1083
1084     :param table: Table to generate.
1085     :param input_data: Data to process.
1086     :type table: dict
1087     :type input_data: InputData
1088     """
1089
1090     _ = input_data
1091
1092     if not table.get(u"testbed", None):
1093         logging.error(
1094             f"The testbed is not defined for the table "
1095             f"{table.get(u'title', u'')}."
1096         )
1097         return
1098
1099     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1100
1101     try:
1102         with open(table[u"input-file"], u'rt') as csv_file:
1103             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1104     except KeyError:
1105         logging.warning(u"The input file is not defined.")
1106         return
1107     except csv.Error as err:
1108         logging.warning(
1109             f"Not possible to process the file {table[u'input-file']}.\n"
1110             f"{repr(err)}"
1111         )
1112         return
1113
1114     # Table:
1115     dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1116
1117     # Table header:
1118     trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1119     for idx, item in enumerate(csv_lst[0]):
1120         alignment = u"left" if idx == 0 else u"center"
1121         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1122         thead.text = item
1123
1124     # Rows:
1125     colors = {
1126         u"regression": (
1127             u"#ffcccc",
1128             u"#ff9999"
1129         ),
1130         u"progression": (
1131             u"#c6ecc6",
1132             u"#9fdf9f"
1133         ),
1134         u"normal": (
1135             u"#e9f1fb",
1136             u"#d4e4f7"
1137         )
1138     }
1139     for r_idx, row in enumerate(csv_lst[1:]):
1140         if int(row[4]):
1141             color = u"regression"
1142         elif int(row[5]):
1143             color = u"progression"
1144         else:
1145             color = u"normal"
1146         trow = ET.SubElement(
1147             dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
1148         )
1149
1150         # Columns:
1151         for c_idx, item in enumerate(row):
1152             tdata = ET.SubElement(
1153                 trow,
1154                 u"td",
1155                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1156             )
1157             # Name:
1158             if c_idx == 0 and table.get(u"add-links", True):
1159                 ref = ET.SubElement(
1160                     tdata,
1161                     u"a",
1162                     attrib=dict(
1163                         href=f"../trending/"
1164                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1165                     )
1166                 )
1167                 ref.text = item
1168             else:
1169                 tdata.text = item
1170     try:
1171         with open(table[u"output-file"], u'w') as html_file:
1172             logging.info(f"    Writing file: {table[u'output-file']}")
1173             html_file.write(u".. raw:: html\n\n\t")
1174             html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
1175             html_file.write(u"\n\t<p><br><br></p>\n")
1176     except KeyError:
1177         logging.warning(u"The output file is not defined.")
1178         return
1179
1180
1181 def table_last_failed_tests(table, input_data):
1182     """Generate the table(s) with algorithm: table_last_failed_tests
1183     specified in the specification file.
1184
1185     :param table: Table to generate.
1186     :param input_data: Data to process.
1187     :type table: pandas.Series
1188     :type input_data: InputData
1189     """
1190
1191     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1192
1193     # Transform the data
1194     logging.info(
1195         f"    Creating the data set for the {table.get(u'type', u'')} "
1196         f"{table.get(u'title', u'')}."
1197     )
1198
1199     data = input_data.filter_data(table, continue_on_error=True)
1200
1201     if data is None or data.empty:
1202         logging.warning(
1203             f"    No data for the {table.get(u'type', u'')} "
1204             f"{table.get(u'title', u'')}."
1205         )
1206         return
1207
1208     tbl_list = list()
1209     for job, builds in table[u"data"].items():
1210         for build in builds:
1211             build = str(build)
1212             try:
1213                 version = input_data.metadata(job, build).get(u"version", u"")
1214             except KeyError:
1215                 logging.error(f"Data for {job}: {build} is not present.")
1216                 return
1217             tbl_list.append(build)
1218             tbl_list.append(version)
1219             failed_tests = list()
1220             passed = 0
1221             failed = 0
1222             for tst_data in data[job][build].values:
1223                 if tst_data[u"status"] != u"FAIL":
1224                     passed += 1
1225                     continue
1226                 failed += 1
1227                 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1228                 if not groups:
1229                     continue
1230                 nic = groups.group(0)
1231                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
1232             tbl_list.append(str(passed))
1233             tbl_list.append(str(failed))
1234             tbl_list.extend(failed_tests)
1235
1236     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1237     logging.info(f"    Writing file: {file_name}")
1238     with open(file_name, u"wt") as file_handler:
1239         for test in tbl_list:
1240             file_handler.write(test + u'\n')
1241
1242
1243 def table_failed_tests(table, input_data):
1244     """Generate the table(s) with algorithm: table_failed_tests
1245     specified in the specification file.
1246
1247     :param table: Table to generate.
1248     :param input_data: Data to process.
1249     :type table: pandas.Series
1250     :type input_data: InputData
1251     """
1252
1253     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1254
1255     # Transform the data
1256     logging.info(
1257         f"    Creating the data set for the {table.get(u'type', u'')} "
1258         f"{table.get(u'title', u'')}."
1259     )
1260     data = input_data.filter_data(table, continue_on_error=True)
1261
1262     # Prepare the header of the tables
1263     header = [
1264         u"Test Case",
1265         u"Failures [#]",
1266         u"Last Failure [Time]",
1267         u"Last Failure [VPP-Build-Id]",
1268         u"Last Failure [CSIT-Job-Build-Id]"
1269     ]
1270
1271     # Generate the data for the table according to the model in the table
1272     # specification
1273
1274     now = dt.utcnow()
1275     timeperiod = timedelta(int(table.get(u"window", 7)))
1276
1277     tbl_dict = dict()
1278     for job, builds in table[u"data"].items():
1279         for build in builds:
1280             build = str(build)
1281             for tst_name, tst_data in data[job][build].items():
1282                 if tst_name.lower() in table.get(u"ignore-list", list()):
1283                     continue
1284                 if tbl_dict.get(tst_name, None) is None:
1285                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
1286                     if not groups:
1287                         continue
1288                     nic = groups.group(0)
1289                     tbl_dict[tst_name] = {
1290                         u"name": f"{nic}-{tst_data[u'name']}",
1291                         u"data": OrderedDict()
1292                     }
1293                 try:
1294                     generated = input_data.metadata(job, build).\
1295                         get(u"generated", u"")
1296                     if not generated:
1297                         continue
1298                     then = dt.strptime(generated, u"%Y%m%d %H:%M")
1299                     if (now - then) <= timeperiod:
1300                         tbl_dict[tst_name][u"data"][build] = (
1301                             tst_data[u"status"],
1302                             generated,
1303                             input_data.metadata(job, build).get(u"version",
1304                                                                 u""),
1305                             build
1306                         )
1307                 except (TypeError, KeyError) as err:
1308                     logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
1309
1310     max_fails = 0
1311     tbl_lst = list()
1312     for tst_data in tbl_dict.values():
1313         fails_nr = 0
1314         fails_last_date = u""
1315         fails_last_vpp = u""
1316         fails_last_csit = u""
1317         for val in tst_data[u"data"].values():
1318             if val[0] == u"FAIL":
1319                 fails_nr += 1
1320                 fails_last_date = val[1]
1321                 fails_last_vpp = val[2]
1322                 fails_last_csit = val[3]
1323         if fails_nr:
1324             max_fails = fails_nr if fails_nr > max_fails else max_fails
1325             tbl_lst.append(
1326                 [
1327                     tst_data[u"name"],
1328                     fails_nr,
1329                     fails_last_date,
1330                     fails_last_vpp,
1331                     f"mrr-daily-build-{fails_last_csit}"
1332                 ]
1333             )
1334
1335     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1336     tbl_sorted = list()
1337     for nrf in range(max_fails, -1, -1):
1338         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1339         tbl_sorted.extend(tbl_fails)
1340
1341     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1342     logging.info(f"    Writing file: {file_name}")
1343     with open(file_name, u"wt") as file_handler:
1344         file_handler.write(u",".join(header) + u"\n")
1345         for test in tbl_sorted:
1346             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1347
1348     logging.info(f"    Writing file: {table[u'output-file']}.txt")
1349     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1350
1351
1352 def table_failed_tests_html(table, input_data):
1353     """Generate the table(s) with algorithm: table_failed_tests_html
1354     specified in the specification file.
1355
1356     :param table: Table to generate.
1357     :param input_data: Data to process.
1358     :type table: pandas.Series
1359     :type input_data: InputData
1360     """
1361
1362     _ = input_data
1363
1364     if not table.get(u"testbed", None):
1365         logging.error(
1366             f"The testbed is not defined for the table "
1367             f"{table.get(u'title', u'')}."
1368         )
1369         return
1370
1371     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1372
1373     try:
1374         with open(table[u"input-file"], u'rt') as csv_file:
1375             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1376     except KeyError:
1377         logging.warning(u"The input file is not defined.")
1378         return
1379     except csv.Error as err:
1380         logging.warning(
1381             f"Not possible to process the file {table[u'input-file']}.\n"
1382             f"{repr(err)}"
1383         )
1384         return
1385
1386     # Table:
1387     failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1388
1389     # Table header:
1390     trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1391     for idx, item in enumerate(csv_lst[0]):
1392         alignment = u"left" if idx == 0 else u"center"
1393         thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1394         thead.text = item
1395
1396     # Rows:
1397     colors = (u"#e9f1fb", u"#d4e4f7")
1398     for r_idx, row in enumerate(csv_lst[1:]):
1399         background = colors[r_idx % 2]
1400         trow = ET.SubElement(
1401             failed_tests, u"tr", attrib=dict(bgcolor=background)
1402         )
1403
1404         # Columns:
1405         for c_idx, item in enumerate(row):
1406             tdata = ET.SubElement(
1407                 trow,
1408                 u"td",
1409                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1410             )
1411             # Name:
1412             if c_idx == 0:
1413                 ref = ET.SubElement(
1414                     tdata,
1415                     u"a",
1416                     attrib=dict(
1417                         href=f"../trending/"
1418                              f"{_generate_url(table.get(u'testbed', ''), item)}"
1419                     )
1420                 )
1421                 ref.text = item
1422             else:
1423                 tdata.text = item
1424     try:
1425         with open(table[u"output-file"], u'w') as html_file:
1426             logging.info(f"    Writing file: {table[u'output-file']}")
1427             html_file.write(u".. raw:: html\n\n\t")
1428             html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
1429             html_file.write(u"\n\t<p><br><br></p>\n")
1430     except KeyError:
1431         logging.warning(u"The output file is not defined.")
1432         return
1433
1434
1435 def table_comparison(table, input_data):
1436     """Generate the table(s) with algorithm: table_comparison
1437     specified in the specification file.
1438
1439     :param table: Table to generate.
1440     :param input_data: Data to process.
1441     :type table: pandas.Series
1442     :type input_data: InputData
1443     """
1444     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1445
1446     # Transform the data
1447     logging.info(
1448         f"    Creating the data set for the {table.get(u'type', u'')} "
1449         f"{table.get(u'title', u'')}."
1450     )
1451
1452     columns = table.get(u"columns", None)
1453     if not columns:
1454         logging.error(
1455             f"No columns specified for {table.get(u'title', u'')}. Skipping."
1456         )
1457         return
1458
1459     cols = list()
1460     for idx, col in enumerate(columns):
1461         if col.get(u"data-set", None) is None:
1462             logging.warning(f"No data for column {col.get(u'title', u'')}")
1463             continue
1464         tag = col.get(u"tag", None)
1465         data = input_data.filter_data(
1466             table,
1467             params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1468             data=col[u"data-set"],
1469             continue_on_error=True
1470         )
1471         col_data = {
1472             u"title": col.get(u"title", f"Column{idx}"),
1473             u"data": dict()
1474         }
1475         for builds in data.values:
1476             for build in builds:
1477                 for tst_name, tst_data in build.items():
1478                     if tag and tag not in tst_data[u"tags"]:
1479                         continue
1480                     tst_name_mod = \
1481                         _tpc_modify_test_name(tst_name, ignore_nic=True).\
1482                         replace(u"2n1l-", u"")
1483                     if col_data[u"data"].get(tst_name_mod, None) is None:
1484                         name = tst_data[u'name'].rsplit(u'-', 1)[0]
1485                         if u"across testbeds" in table[u"title"].lower() or \
1486                                 u"across topologies" in table[u"title"].lower():
1487                             name = _tpc_modify_displayed_test_name(name)
1488                         col_data[u"data"][tst_name_mod] = {
1489                             u"name": name,
1490                             u"replace": True,
1491                             u"data": list(),
1492                             u"mean": None,
1493                             u"stdev": None
1494                         }
1495                     _tpc_insert_data(
1496                         target=col_data[u"data"][tst_name_mod],
1497                         src=tst_data,
1498                         include_tests=table[u"include-tests"]
1499                     )
1500
1501         replacement = col.get(u"data-replacement", None)
1502         if replacement:
1503             rpl_data = input_data.filter_data(
1504                 table,
1505                 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1506                 data=replacement,
1507                 continue_on_error=True
1508             )
1509             for builds in rpl_data.values:
1510                 for build in builds:
1511                     for tst_name, tst_data in build.items():
1512                         if tag and tag not in tst_data[u"tags"]:
1513                             continue
1514                         tst_name_mod = \
1515                             _tpc_modify_test_name(tst_name, ignore_nic=True).\
1516                             replace(u"2n1l-", u"")
1517                         if col_data[u"data"].get(tst_name_mod, None) is None:
1518                             name = tst_data[u'name'].rsplit(u'-', 1)[0]
1519                             if u"across testbeds" in table[u"title"].lower() \
1520                                     or u"across topologies" in \
1521                                     table[u"title"].lower():
1522                                 name = _tpc_modify_displayed_test_name(name)
1523                             col_data[u"data"][tst_name_mod] = {
1524                                 u"name": name,
1525                                 u"replace": False,
1526                                 u"data": list(),
1527                                 u"mean": None,
1528                                 u"stdev": None
1529                             }
1530                         if col_data[u"data"][tst_name_mod][u"replace"]:
1531                             col_data[u"data"][tst_name_mod][u"replace"] = False
1532                             col_data[u"data"][tst_name_mod][u"data"] = list()
1533                         _tpc_insert_data(
1534                             target=col_data[u"data"][tst_name_mod],
1535                             src=tst_data,
1536                             include_tests=table[u"include-tests"]
1537                         )
1538
1539         if table[u"include-tests"] in (u"NDR", u"PDR"):
1540             for tst_name, tst_data in col_data[u"data"].items():
1541                 if tst_data[u"data"]:
1542                     tst_data[u"mean"] = mean(tst_data[u"data"])
1543                     tst_data[u"stdev"] = stdev(tst_data[u"data"])
1544
1545         cols.append(col_data)
1546
1547     tbl_dict = dict()
1548     for col in cols:
1549         for tst_name, tst_data in col[u"data"].items():
1550             if tbl_dict.get(tst_name, None) is None:
1551                 tbl_dict[tst_name] = {
1552                     "name": tst_data[u"name"]
1553                 }
1554             tbl_dict[tst_name][col[u"title"]] = {
1555                 u"mean": tst_data[u"mean"],
1556                 u"stdev": tst_data[u"stdev"]
1557             }
1558
1559     if not tbl_dict:
1560         logging.warning(f"No data for table {table.get(u'title', u'')}!")
1561         return
1562
1563     tbl_lst = list()
1564     for tst_data in tbl_dict.values():
1565         row = [tst_data[u"name"], ]
1566         for col in cols:
1567             row.append(tst_data.get(col[u"title"], None))
1568         tbl_lst.append(row)
1569
1570     comparisons = table.get(u"comparisons", None)
1571     if comparisons and isinstance(comparisons, list):
1572         for idx, comp in enumerate(comparisons):
1573             try:
1574                 col_ref = int(comp[u"reference"])
1575                 col_cmp = int(comp[u"compare"])
1576             except KeyError:
1577                 logging.warning(u"Comparison: No references defined! Skipping.")
1578                 comparisons.pop(idx)
1579                 continue
1580             if not (0 < col_ref <= len(cols) and
1581                     0 < col_cmp <= len(cols)) or \
1582                     col_ref == col_cmp:
1583                 logging.warning(f"Wrong values of reference={col_ref} "
1584                                 f"and/or compare={col_cmp}. Skipping.")
1585                 comparisons.pop(idx)
1586                 continue
1587
1588     tbl_cmp_lst = list()
1589     if comparisons:
1590         for row in tbl_lst:
1591             new_row = deepcopy(row)
1592             add_to_tbl = False
1593             for comp in comparisons:
1594                 ref_itm = row[int(comp[u"reference"])]
1595                 if ref_itm is None and \
1596                         comp.get(u"reference-alt", None) is not None:
1597                     ref_itm = row[int(comp[u"reference-alt"])]
1598                 cmp_itm = row[int(comp[u"compare"])]
1599                 if ref_itm is not None and cmp_itm is not None and \
1600                         ref_itm[u"mean"] is not None and \
1601                         cmp_itm[u"mean"] is not None and \
1602                         ref_itm[u"stdev"] is not None and \
1603                         cmp_itm[u"stdev"] is not None:
1604                     delta, d_stdev = relative_change_stdev(
1605                         ref_itm[u"mean"], cmp_itm[u"mean"],
1606                         ref_itm[u"stdev"], cmp_itm[u"stdev"]
1607                     )
1608                     new_row.append(
1609                         {
1610                             u"mean": delta * 1e6,
1611                             u"stdev": d_stdev * 1e6
1612                         }
1613                     )
1614                     add_to_tbl = True
1615                 else:
1616                     new_row.append(None)
1617             if add_to_tbl:
1618                 tbl_cmp_lst.append(new_row)
1619
1620     tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
1621     tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
1622
1623     rcas = list()
1624     rca_in = table.get(u"rca", None)
1625     if rca_in and isinstance(rca_in, list):
1626         for idx, itm in enumerate(rca_in):
1627             try:
1628                 with open(itm.get(u"data", u""), u"r") as rca_file:
1629                     rcas.append(
1630                         {
1631                             u"title": itm.get(u"title", f"RCA{idx}"),
1632                             u"data": load(rca_file, Loader=FullLoader)
1633                         }
1634                     )
1635             except (YAMLError, IOError) as err:
1636                 logging.warning(
1637                     f"The RCA file {itm.get(u'data', u'')} does not exist or "
1638                     f"it is corrupted!"
1639                 )
1640                 logging.debug(repr(err))
1641
1642     tbl_for_csv = list()
1643     for line in tbl_cmp_lst:
1644         row = [line[0], ]
1645         for idx, itm in enumerate(line[1:]):
1646             if itm is None:
1647                 row.append(u"NT")
1648                 row.append(u"NT")
1649             else:
1650                 row.append(round(float(itm[u'mean']) / 1e6, 3))
1651                 row.append(round(float(itm[u'stdev']) / 1e6, 3))
1652         for rca in rcas:
1653             rca_nr = rca[u"data"].get(row[0], u"-")
1654             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1655         tbl_for_csv.append(row)
1656
1657     header_csv = [u"Test Case", ]
1658     for col in cols:
1659         header_csv.append(f"Avg({col[u'title']})")
1660         header_csv.append(f"Stdev({col[u'title']})")
1661     for comp in comparisons:
1662         header_csv.append(
1663             f"Avg({comp.get(u'title', u'')})"
1664         )
1665         header_csv.append(
1666             f"Stdev({comp.get(u'title', u'')})"
1667         )
1668     header_csv.extend([rca[u"title"] for rca in rcas])
1669
1670     legend_lst = table.get(u"legend", None)
1671     if legend_lst is None:
1672         legend = u""
1673     else:
1674         legend = u"\n" + u"\n".join(legend_lst) + u"\n"
1675
1676     footnote = u""
1677     for rca in rcas:
1678         footnote += f"\n{rca[u'title']}:\n"
1679         footnote += rca[u"data"].get(u"footnote", u"")
1680
1681     csv_file = f"{table[u'output-file']}-csv.csv"
1682     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1683         file_handler.write(
1684             u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
1685         )
1686         for test in tbl_for_csv:
1687             file_handler.write(
1688                 u",".join([f'"{item}"' for item in test]) + u"\n"
1689             )
1690         if legend_lst:
1691             for item in legend_lst:
1692                 file_handler.write(f'"{item}"\n')
1693         if footnote:
1694             for itm in footnote.split(u"\n"):
1695                 file_handler.write(f'"{itm}"\n')
1696
1697     tbl_tmp = list()
1698     max_lens = [0, ] * len(tbl_cmp_lst[0])
1699     for line in tbl_cmp_lst:
1700         row = [line[0], ]
1701         for idx, itm in enumerate(line[1:]):
1702             if itm is None:
1703                 new_itm = u"NT"
1704             else:
1705                 if idx < len(cols):
1706                     new_itm = (
1707                         f"{round(float(itm[u'mean']) / 1e6, 1)} "
1708                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1709                         replace(u"nan", u"NaN")
1710                     )
1711                 else:
1712                     new_itm = (
1713                         f"{round(float(itm[u'mean']) / 1e6, 1):+} "
1714                         f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1715                         replace(u"nan", u"NaN")
1716                     )
1717             if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]:
1718                 max_lens[idx] = len(new_itm.rsplit(u" ", 1)[-1])
1719             row.append(new_itm)
1720
1721         tbl_tmp.append(row)
1722
1723     tbl_final = list()
1724     for line in tbl_tmp:
1725         row = [line[0], ]
1726         for idx, itm in enumerate(line[1:]):
1727             if itm in (u"NT", u"NaN"):
1728                 row.append(itm)
1729                 continue
1730             itm_lst = itm.rsplit(u"\u00B1", 1)
1731             itm_lst[-1] = \
1732                 f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
1733             row.append(u"\u00B1".join(itm_lst))
1734         for rca in rcas:
1735             rca_nr = rca[u"data"].get(row[0], u"-")
1736             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1737
1738         tbl_final.append(row)
1739
1740     header = [u"Test Case", ]
1741     header.extend([col[u"title"] for col in cols])
1742     header.extend([comp.get(u"title", u"") for comp in comparisons])
1743     header.extend([rca[u"title"] for rca in rcas])
1744
1745     # Generate csv tables:
1746     csv_file = f"{table[u'output-file']}.csv"
1747     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1748         file_handler.write(u";".join(header) + u"\n")
1749         for test in tbl_final:
1750             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1751
1752     # Generate txt table:
1753     txt_file_name = f"{table[u'output-file']}.txt"
1754     convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
1755
1756     with open(txt_file_name, u'a', encoding='utf-8') as txt_file:
1757         txt_file.write(legend)
1758         txt_file.write(footnote)
1759         if legend or footnote:
1760             txt_file.write(u"\n:END")
1761
1762     # Generate html table:
1763     _tpc_generate_html_table(
1764         header,
1765         tbl_final,
1766         table[u'output-file'],
1767         legend=legend,
1768         footnote=footnote,
1769         sort_data=False,
1770         title=table.get(u"title", u"")
1771     )
1772
1773
1774 def table_weekly_comparison(table, in_data):
1775     """Generate the table(s) with algorithm: table_weekly_comparison
1776     specified in the specification file.
1777
1778     :param table: Table to generate.
1779     :param in_data: Data to process.
1780     :type table: pandas.Series
1781     :type in_data: InputData
1782     """
1783     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
1784
1785     # Transform the data
1786     logging.info(
1787         f"    Creating the data set for the {table.get(u'type', u'')} "
1788         f"{table.get(u'title', u'')}."
1789     )
1790
1791     incl_tests = table.get(u"include-tests", None)
1792     if incl_tests not in (u"NDR", u"PDR"):
1793         logging.error(f"Wrong tests to include specified ({incl_tests}).")
1794         return
1795
1796     nr_cols = table.get(u"nr-of-data-columns", None)
1797     if not nr_cols or nr_cols < 2:
1798         logging.error(
1799             f"No columns specified for {table.get(u'title', u'')}. Skipping."
1800         )
1801         return
1802
1803     data = in_data.filter_data(
1804         table,
1805         params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1806         continue_on_error=True
1807     )
1808
1809     header = [
1810         [u"VPP Version", ],
1811         [u"Start Timestamp", ],
1812         [u"CSIT Build", ],
1813         [u"CSIT Testbed", ]
1814     ]
1815     tbl_dict = dict()
1816     idx = 0
1817     tb_tbl = table.get(u"testbeds", None)
1818     for job_name, job_data in data.items():
1819         for build_nr, build in job_data.items():
1820             if idx >= nr_cols:
1821                 break
1822             if build.empty:
1823                 continue
1824
1825             tb_ip = in_data.metadata(job_name, build_nr).get(u"testbed", u"")
1826             if tb_ip and tb_tbl:
1827                 testbed = tb_tbl.get(tb_ip, u"")
1828             else:
1829                 testbed = u""
1830             header[2].insert(1, build_nr)
1831             header[3].insert(1, testbed)
1832             header[1].insert(
1833                 1, in_data.metadata(job_name, build_nr).get(u"generated", u"")
1834             )
1835             header[0].insert(
1836                 1, in_data.metadata(job_name, build_nr).get(u"version", u"")
1837             )
1838
1839             for tst_name, tst_data in build.items():
1840                 tst_name_mod = \
1841                     _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
1842                 if not tbl_dict.get(tst_name_mod, None):
1843                     tbl_dict[tst_name_mod] = dict(
1844                         name=tst_data[u'name'].rsplit(u'-', 1)[0],
1845                     )
1846                 try:
1847                     tbl_dict[tst_name_mod][-idx - 1] = \
1848                         tst_data[u"throughput"][incl_tests][u"LOWER"]
1849                 except (TypeError, IndexError, KeyError, ValueError):
1850                     pass
1851             idx += 1
1852
1853     if idx < nr_cols:
1854         logging.error(u"Not enough data to build the table! Skipping")
1855         return
1856
1857     cmp_dict = dict()
1858     for idx, cmp in enumerate(table.get(u"comparisons", list())):
1859         idx_ref = cmp.get(u"reference", None)
1860         idx_cmp = cmp.get(u"compare", None)
1861         if idx_ref is None or idx_cmp is None:
1862             continue
1863         header[0].append(
1864             f"Diff({header[0][idx_ref - idx].split(u'~')[-1]} vs "
1865             f"{header[0][idx_cmp - idx].split(u'~')[-1]})"
1866         )
1867         header[1].append(u"")
1868         header[2].append(u"")
1869         header[3].append(u"")
1870         for tst_name, tst_data in tbl_dict.items():
1871             if not cmp_dict.get(tst_name, None):
1872                 cmp_dict[tst_name] = list()
1873             ref_data = tst_data.get(idx_ref, None)
1874             cmp_data = tst_data.get(idx_cmp, None)
1875             if ref_data is None or cmp_data is None:
1876                 cmp_dict[tst_name].append(float('nan'))
1877             else:
1878                 cmp_dict[tst_name].append(
1879                     relative_change(ref_data, cmp_data)
1880                 )
1881
1882     tbl_lst = list()
1883     for tst_name, tst_data in tbl_dict.items():
1884         itm_lst = [tst_data[u"name"], ]
1885         for idx in range(nr_cols):
1886             item = tst_data.get(-idx - 1, None)
1887             if item is None:
1888                 itm_lst.insert(1, None)
1889             else:
1890                 itm_lst.insert(1, round(item / 1e6, 1))
1891         itm_lst.extend(
1892             [
1893                 None if itm is None else round(itm, 1)
1894                 for itm in cmp_dict[tst_name]
1895             ]
1896         )
1897         tbl_lst.append(itm_lst)
1898
1899     tbl_lst.sort(key=lambda rel: rel[0], reverse=False)
1900     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1901
1902     # Generate csv table:
1903     csv_file = f"{table[u'output-file']}.csv"
1904     with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1905         for hdr in header:
1906             file_handler.write(u",".join(hdr) + u"\n")
1907         for test in tbl_lst:
1908             file_handler.write(u",".join(
1909                 [
1910                     str(item).replace(u"None", u"-").replace(u"nan", u"-").
1911                     replace(u"null", u"-") for item in test
1912                 ]
1913             ) + u"\n")
1914
1915     txt_file = f"{table[u'output-file']}.txt"
1916     convert_csv_to_pretty_txt(csv_file, txt_file, delimiter=u",")
1917
1918     # Reorganize header in txt table
1919     txt_table = list()
1920     with open(txt_file, u"rt", encoding='utf-8') as file_handler:
1921         for line in file_handler:
1922             txt_table.append(line)
1923     try:
1924         txt_table.insert(5, txt_table.pop(2))
1925         with open(txt_file, u"wt", encoding='utf-8') as file_handler:
1926             file_handler.writelines(txt_table)
1927     except IndexError:
1928         pass
1929
1930     # Generate html table:
1931     hdr_html = [
1932         u"<br>".join(row) for row in zip(*header)
1933     ]
1934     _tpc_generate_html_table(
1935         hdr_html,
1936         tbl_lst,
1937         table[u'output-file'],
1938         sort_data=True,
1939         title=table.get(u"title", u""),
1940         generate_rst=False
1941     )