X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=449b2357a8aa7e5165030d228b5bc79e7685b5a6;hp=0afbf87ac04a357386647e5d1c2b6f7d69bc09d7;hb=1da19da813655f643bc3c6e4d03bed987f076f07;hpb=d6c14f8b47849a4885aac6dad5c8d19baad4b9c3
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 0afbf87ac0..449b2357a8 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
@@ -23,18 +23,21 @@ from collections import OrderedDict
from xml.etree import ElementTree as ET
from datetime import datetime as dt
from datetime import timedelta
+from copy import deepcopy
+from json import loads
import plotly.graph_objects as go
import plotly.offline as ploff
import pandas as pd
from numpy import nan, isnan
+from yaml import load, FullLoader, YAMLError
-from pal_utils import mean, stdev, relative_change, classify_anomalies, \
- convert_csv_to_pretty_txt, relative_change_stdev
+from pal_utils import mean, stdev, classify_anomalies, \
+ convert_csv_to_pretty_txt, relative_change_stdev, relative_change
-REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*')
+REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
def generate_tables(spec, data):
@@ -47,22 +50,23 @@ def generate_tables(spec, data):
"""
generator = {
- u"table_details": table_details,
u"table_merged_details": table_merged_details,
- u"table_perf_comparison": table_perf_comparison,
- u"table_perf_comparison_nic": table_perf_comparison_nic,
- u"table_nics_comparison": table_nics_comparison,
u"table_soak_vs_ndr": table_soak_vs_ndr,
u"table_perf_trending_dash": table_perf_trending_dash,
u"table_perf_trending_dash_html": table_perf_trending_dash_html,
u"table_last_failed_tests": table_last_failed_tests,
u"table_failed_tests": table_failed_tests,
- u"table_failed_tests_html": table_failed_tests_html
+ u"table_failed_tests_html": table_failed_tests_html,
+ u"table_oper_data_html": table_oper_data_html,
+ u"table_comparison": table_comparison,
+ u"table_weekly_comparison": table_weekly_comparison
}
logging.info(u"Generating the tables ...")
for table in spec.tables:
try:
+ if table[u"algorithm"] == u"table_weekly_comparison":
+ table[u"testbeds"] = spec.environment.get(u"testbeds", None)
generator[table[u"algorithm"]](table, data)
except NameError as err:
logging.error(
@@ -72,8 +76,8 @@ def generate_tables(spec, data):
logging.info(u"Done.")
-def table_details(table, input_data):
- """Generate the table(s) with algorithm: table_detailed_test_results
+def table_oper_data_html(table, input_data):
+ """Generate the table(s) with algorithm: html_table_oper_data
specified in the specification file.
:param table: Table to generate.
@@ -83,66 +87,236 @@ def table_details(table, input_data):
"""
logging.info(f" Generating the table {table.get(u'title', u'')} ...")
-
# Transform the data
logging.info(
f" Creating the data set for the {table.get(u'type', u'')} "
f"{table.get(u'title', u'')}."
)
- data = input_data.filter_data(table)
+ data = input_data.filter_data(
+ table,
+ params=[u"name", u"parent", u"telemetry-show-run", u"type"],
+ continue_on_error=True
+ )
+ if data.empty:
+ return
+ data = input_data.merge_data(data)
- # Prepare the header of the tables
- header = list()
- for column in table[u"columns"]:
- header.append(
- u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
+ sort_tests = table.get(u"sort", None)
+ if sort_tests:
+ args = dict(
+ inplace=True,
+ ascending=(sort_tests == u"ascending")
)
+ data.sort_index(**args)
- # Generate the data for the table according to the model in the table
- # specification
- job = list(table[u"data"].keys())[0]
- build = str(table[u"data"][job][0])
- try:
- suites = input_data.suites(job, build)
- except KeyError:
- logging.error(
- u" No data available. The table will not be generated."
- )
+ suites = input_data.filter_data(
+ table,
+ continue_on_error=True,
+ data_set=u"suites"
+ )
+ if suites.empty:
return
+ suites = input_data.merge_data(suites)
- for suite in suites.values:
- # Generate data
- suite_name = suite[u"name"]
- table_lst = list()
- for test in data[job][build].keys():
- if data[job][build][test][u"parent"] not in suite_name:
+ def _generate_html_table(tst_data):
+ """Generate an HTML table with operational data for the given test.
+
+ :param tst_data: Test data to be used to generate the table.
+ :type tst_data: pandas.Series
+ :returns: HTML table with operational data.
+ :rtype: str
+ """
+
+ colors = {
+ u"header": u"#7eade7",
+ u"empty": u"#ffffff",
+ u"body": (u"#e9f1fb", u"#d4e4f7")
+ }
+
+ tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
+
+ trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
+ thead = ET.SubElement(
+ trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+ )
+ thead.text = tst_data[u"name"]
+
+ trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
+ thead = ET.SubElement(
+ trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+ )
+ thead.text = u"\t"
+
+ if tst_data.get(u"telemetry-show-run", None) is None or \
+ isinstance(tst_data[u"telemetry-show-run"], str):
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
+ )
+ tcol = ET.SubElement(
+ trow, u"td", attrib=dict(align=u"left", colspan=u"6")
+ )
+ tcol.text = u"No Data"
+
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
+ )
+ thead = ET.SubElement(
+ trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+ )
+ font = ET.SubElement(
+ thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
+ )
+ font.text = u"."
+ return str(ET.tostring(tbl, encoding=u"unicode"))
+
+ tbl_hdr = (
+ u"Name",
+ u"Nr of Vectors",
+ u"Nr of Packets",
+ u"Suspends",
+ u"Cycles per Packet",
+ u"Average Vector Size"
+ )
+
+ for dut_data in tst_data[u"telemetry-show-run"].values():
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
+ )
+ tcol = ET.SubElement(
+ trow, u"td", attrib=dict(align=u"left", colspan=u"6")
+ )
+ if dut_data.get(u"runtime", None) is None:
+ tcol.text = u"No Data"
continue
- row_lst = list()
- for column in table[u"columns"]:
- try:
- col_data = str(data[job][build][test][column[
- u"data"].split(" ")[1]]).replace(u'"', u'""')
- if column[u"data"].split(u" ")[1] in \
- (u"conf-history", u"show-run"):
- col_data = col_data.replace(u" |br| ", u"", 1)
- col_data = f" |prein| {col_data[:-5]} |preout| "
- row_lst.append(f'"{col_data}"')
- except KeyError:
- row_lst.append(u"No data")
- table_lst.append(row_lst)
- # Write the data to file
- if table_lst:
- file_name = (
- f"{table[u'output-file']}_{suite_name}"
- f"{table[u'output-file-ext']}"
+ runtime = dict()
+ for item in dut_data[u"runtime"].get(u"data", tuple()):
+ tid = int(item[u"labels"][u"thread_id"])
+ if runtime.get(tid, None) is None:
+ runtime[tid] = dict()
+ gnode = item[u"labels"][u"graph_node"]
+ if runtime[tid].get(gnode, None) is None:
+ runtime[tid][gnode] = dict()
+ try:
+ runtime[tid][gnode][item[u"name"]] = float(item[u"value"])
+ except ValueError:
+ runtime[tid][gnode][item[u"name"]] = item[u"value"]
+
+ threads = dict({idx: list() for idx in range(len(runtime))})
+ for idx, run_data in runtime.items():
+ for gnode, gdata in run_data.items():
+ if gdata[u"vectors"] > 0:
+ clocks = gdata[u"clocks"] / gdata[u"vectors"]
+ elif gdata[u"calls"] > 0:
+ clocks = gdata[u"clocks"] / gdata[u"calls"]
+ elif gdata[u"suspends"] > 0:
+ clocks = gdata[u"clocks"] / gdata[u"suspends"]
+ else:
+ clocks = 0.0
+ if gdata[u"calls"] > 0:
+ vectors_call = gdata[u"vectors"] / gdata[u"calls"]
+ else:
+ vectors_call = 0.0
+ if int(gdata[u"calls"]) + int(gdata[u"vectors"]) + \
+ int(gdata[u"suspends"]):
+ threads[idx].append([
+ gnode,
+ int(gdata[u"calls"]),
+ int(gdata[u"vectors"]),
+ int(gdata[u"suspends"]),
+ clocks,
+ vectors_call
+ ])
+
+ bold = ET.SubElement(tcol, u"b")
+ bold.text = (
+ f"Host IP: {dut_data.get(u'host', '')}, "
+ f"Socket: {dut_data.get(u'socket', '')}"
)
- logging.info(f" Writing file: {file_name}")
- with open(file_name, u"wt") as file_handler:
- file_handler.write(u",".join(header) + u"\n")
- for item in table_lst:
- file_handler.write(u",".join(item) + u"\n")
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
+ )
+ thead = ET.SubElement(
+ trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+ )
+ thead.text = u"\t"
+
+ for thread_nr, thread in threads.items():
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
+ )
+ tcol = ET.SubElement(
+ trow, u"td", attrib=dict(align=u"left", colspan=u"6")
+ )
+ bold = ET.SubElement(tcol, u"b")
+ bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
+ )
+ for idx, col in enumerate(tbl_hdr):
+ tcol = ET.SubElement(
+ trow, u"td",
+ attrib=dict(align=u"right" if idx else u"left")
+ )
+ font = ET.SubElement(
+ tcol, u"font", attrib=dict(size=u"2")
+ )
+ bold = ET.SubElement(font, u"b")
+ bold.text = col
+ for row_nr, row in enumerate(thread):
+ trow = ET.SubElement(
+ tbl, u"tr",
+ attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
+ )
+ for idx, col in enumerate(row):
+ tcol = ET.SubElement(
+ trow, u"td",
+ attrib=dict(align=u"right" if idx else u"left")
+ )
+ font = ET.SubElement(
+ tcol, u"font", attrib=dict(size=u"2")
+ )
+ if isinstance(col, float):
+ font.text = f"{col:.2f}"
+ else:
+ font.text = str(col)
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
+ )
+ thead = ET.SubElement(
+ trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+ )
+ thead.text = u"\t"
+ trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
+ thead = ET.SubElement(
+ trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+ )
+ font = ET.SubElement(
+ thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
+ )
+ font.text = u"."
+
+ return str(ET.tostring(tbl, encoding=u"unicode"))
+
+ for suite in suites.values:
+ html_table = str()
+ for test_data in data.values:
+ if test_data[u"parent"] not in suite[u"name"]:
+ continue
+ html_table += _generate_html_table(test_data)
+ if not html_table:
+ continue
+ try:
+ file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
+ with open(f"{file_name}", u'w') as html_file:
+ logging.info(f" Writing file: {file_name}")
+ html_file.write(u".. raw:: html\n\n\t")
+ html_file.write(html_table)
+ html_file.write(u"\n\t
\n")
+ except KeyError:
+ logging.warning(u"The output file is not defined.")
+ return
logging.info(u" Done.")
@@ -157,6 +331,7 @@ def table_merged_details(table, input_data):
"""
logging.info(f" Generating the table {table.get(u'title', u'')} ...")
+
# Transform the data
logging.info(
f" Creating the data set for the {table.get(u'type', u'')} "
@@ -164,12 +339,15 @@ def table_merged_details(table, input_data):
)
data = input_data.filter_data(table, continue_on_error=True)
data = input_data.merge_data(data)
- data.sort_index(inplace=True)
- logging.info(
- f" Creating the data set for the {table.get(u'type', u'')} "
- f"{table.get(u'title', u'')}."
- )
+ sort_tests = table.get(u"sort", None)
+ if sort_tests:
+ args = dict(
+ inplace=True,
+ ascending=(sort_tests == u"ascending")
+ )
+ data.sort_index(**args)
+
suites = input_data.filter_data(
table, continue_on_error=True, data_set=u"suites")
suites = input_data.merge_data(suites)
@@ -186,31 +364,51 @@ def table_merged_details(table, input_data):
suite_name = suite[u"name"]
table_lst = list()
for test in data.keys():
- if data[test][u"parent"] not in suite_name:
+ if data[test][u"status"] != u"PASS" or \
+ data[test][u"parent"] not in suite_name:
continue
row_lst = list()
for column in table[u"columns"]:
try:
col_data = str(data[test][column[
u"data"].split(u" ")[1]]).replace(u'"', u'""')
+ # Do not include tests with "Test Failed" in test message
+ if u"Test Failed" in col_data:
+ continue
col_data = col_data.replace(
u"No Data", u"Not Captured "
)
- if column[u"data"].split(u" ")[1] in \
- (u"conf-history", u"show-run"):
- col_data = col_data.replace(u" |br| ", u"", 1)
+ if column[u"data"].split(u" ")[1] in (u"name", ):
+ if len(col_data) > 30:
+ col_data_lst = col_data.split(u"-")
+ half = int(len(col_data_lst) / 2)
+ col_data = f"{u'-'.join(col_data_lst[:half])}" \
+ f"- |br| " \
+ f"{u'-'.join(col_data_lst[half:])}"
+ col_data = f" |prein| {col_data} |preout| "
+ elif column[u"data"].split(u" ")[1] in (u"msg", ):
+ # Temporary solution: remove NDR results from message:
+ if bool(table.get(u'remove-ndr', False)):
+ try:
+ col_data = col_data.split(u"\n", 1)[1]
+ except IndexError:
+ pass
+ col_data = col_data.replace(u'\n', u' |br| ').\
+ replace(u'\r', u'').replace(u'"', u"'")
+ col_data = f" |prein| {col_data} |preout| "
+ elif column[u"data"].split(u" ")[1] in (u"conf-history", ):
+ col_data = col_data.replace(u'\n', u' |br| ')
col_data = f" |prein| {col_data[:-5]} |preout| "
row_lst.append(f'"{col_data}"')
except KeyError:
row_lst.append(u'"Not captured"')
- table_lst.append(row_lst)
+ if len(row_lst) == len(table[u"columns"]):
+ table_lst.append(row_lst)
# Write the data to file
if table_lst:
- file_name = (
- f"{table[u'output-file']}_{suite_name}"
- f"{table[u'output-file-ext']}"
- )
+ separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
+ file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
logging.info(f" Writing file: {file_name}")
with open(file_name, u"wt") as file_handler:
file_handler.write(u",".join(header) + u"\n")
@@ -220,21 +418,18 @@ def table_merged_details(table, input_data):
logging.info(u" Done.")
-def _tpc_modify_test_name(test_name):
+def _tpc_modify_test_name(test_name, ignore_nic=False):
"""Modify a test name by replacing its parts.
:param test_name: Test name to be modified.
+ :param ignore_nic: If True, NIC is removed from TC name.
:type test_name: str
+ :type ignore_nic: bool
:returns: Modified test name.
:rtype: str
"""
test_name_mod = test_name.\
- replace(u"-ndrpdrdisc", u""). \
replace(u"-ndrpdr", u"").\
- replace(u"-pdrdisc", u""). \
- replace(u"-ndrdisc", u"").\
- replace(u"-pdr", u""). \
- replace(u"-ndr", u""). \
replace(u"1t1c", u"1c").\
replace(u"2t1c", u"1c"). \
replace(u"2t2c", u"2c").\
@@ -242,7 +437,9 @@ def _tpc_modify_test_name(test_name):
replace(u"4t4c", u"4c").\
replace(u"8t4c", u"4c")
- return re.sub(REGEX_NIC, u"", test_name_mod)
+ if ignore_nic:
+ return re.sub(REGEX_NIC, u"", test_name_mod)
+ return test_name_mod
def _tpc_modify_displayed_test_name(test_name):
@@ -266,7 +463,7 @@ def _tpc_insert_data(target, src, include_tests):
"""Insert src data to the target structure.
:param target: Target structure where the data is placed.
- :param src: Source data to be placed into the target stucture.
+ :param src: Source data to be placed into the target structure.
:param include_tests: Which results will be included (MRR, NDR, PDR).
:type target: list
:type src: dict
@@ -274,791 +471,202 @@ def _tpc_insert_data(target, src, include_tests):
"""
try:
if include_tests == u"MRR":
- target.append(src[u"result"][u"receive-rate"])
+ target[u"mean"] = src[u"result"][u"receive-rate"]
+ target[u"stdev"] = src[u"result"][u"receive-stdev"]
elif include_tests == u"PDR":
- target.append(src[u"throughput"][u"PDR"][u"LOWER"])
+ target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
elif include_tests == u"NDR":
- target.append(src[u"throughput"][u"NDR"][u"LOWER"])
+ target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
except (KeyError, TypeError):
pass
-def _tpc_sort_table(table):
- """Sort the table this way:
-
- 1. Put "New in CSIT-XXXX" at the first place.
- 2. Put "See footnote" at the second place.
- 3. Sort the rest by "Delta".
-
- :param table: Table to sort.
- :type table: list
- :returns: Sorted table.
- :rtype: list
- """
-
-
- tbl_new = list()
- tbl_see = list()
- tbl_delta = list()
- for item in table:
- if isinstance(item[-1], str):
- if u"New in CSIT" in item[-1]:
- tbl_new.append(item)
- elif u"See footnote" in item[-1]:
- tbl_see.append(item)
- else:
- tbl_delta.append(item)
-
- # Sort the tables:
- tbl_new.sort(key=lambda rel: rel[0], reverse=False)
- tbl_see.sort(key=lambda rel: rel[0], reverse=False)
- tbl_see.sort(key=lambda rel: rel[-1], reverse=False)
- tbl_delta.sort(key=lambda rel: rel[-1], reverse=True)
-
- # Put the tables together:
- table = list()
- table.extend(tbl_new)
- table.extend(tbl_see)
- table.extend(tbl_delta)
-
- return table
-
-
-def _tpc_generate_html_table(header, data, output_file_name):
+def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
+ footnote=u"", sort_data=True, title=u"",
+ generate_rst=True):
"""Generate html table from input data with simple sorting possibility.
:param header: Table header.
:param data: Input data to be included in the table. It is a list of lists.
Inner lists are rows in the table. All inner lists must be of the same
- length. The length of these lists must be the same as the length of the
- header.
- :param output_file_name: The name (relative or full path) where the
- generated html table is written.
- :type header: list
- :type data: list of lists
- :type output_file_name: str
- """
-
- df_data = pd.DataFrame(data, columns=header)
-
- df_sorted = [df_data.sort_values(
- by=[key, header[0]], ascending=[True, True]
- if key != header[0] else [False, True]) for key in header]
- df_sorted_rev = [df_data.sort_values(
- by=[key, header[0]], ascending=[False, True]
- if key != header[0] else [True, True]) for key in header]
- df_sorted.extend(df_sorted_rev)
-
- fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
- for idx in range(len(df_data))]]
- table_header = dict(
- values=[f"{item}" for item in header],
- fill_color=u"#7eade7",
- align=[u"left", u"center"]
- )
-
- fig = go.Figure()
-
- for table in df_sorted:
- columns = [table.get(col) for col in header]
- fig.add_trace(
- go.Table(
- columnwidth=[30, 10],
- header=table_header,
- cells=dict(
- values=columns,
- fill_color=fill_color,
- align=[u"left", u"right"]
- )
- )
- )
-
- buttons = list()
- menu_items = [f"{itm} (ascending)" for itm in header]
- menu_items_rev = [f"{itm} (descending)" for itm in header]
- menu_items.extend(menu_items_rev)
- for idx, hdr in enumerate(menu_items):
- visible = [False, ] * len(menu_items)
- visible[idx] = True
- buttons.append(
- dict(
- label=hdr.replace(u" [Mpps]", u""),
- method=u"update",
- args=[{u"visible": visible}],
- )
- )
-
- fig.update_layout(
- updatemenus=[
- go.layout.Updatemenu(
- type=u"dropdown",
- direction=u"down",
- x=0.03,
- xanchor=u"left",
- y=1.045,
- yanchor=u"top",
- active=len(menu_items) - 1,
- buttons=list(buttons)
- )
- ],
- annotations=[
- go.layout.Annotation(
- text=u"Sort by:",
- x=0,
- xref=u"paper",
- y=1.035,
- yref=u"paper",
- align=u"left",
- showarrow=False
- )
- ]
- )
-
- ploff.plot(fig, show_link=False, auto_open=False, filename=output_file_name)
-
-
-def table_perf_comparison(table, input_data):
- """Generate the table(s) with algorithm: table_perf_comparison
- specified in the specification file.
-
- :param table: Table to generate.
- :param input_data: Data to process.
- :type table: pandas.Series
- :type input_data: InputData
- """
-
- logging.info(f" Generating the table {table.get(u'title', u'')} ...")
-
- # Transform the data
- logging.info(
- f" Creating the data set for the {table.get(u'type', u'')} "
- f"{table.get(u'title', u'')}."
- )
- data = input_data.filter_data(table, continue_on_error=True)
-
- # Prepare the header of the tables
- try:
- header = [u"Test case", ]
-
- if table[u"include-tests"] == u"MRR":
- hdr_param = u"Rec Rate"
- else:
- hdr_param = u"Thput"
-
- history = table.get(u"history", list())
- for item in history:
- header.extend(
- [
- f"{item[u'title']} {hdr_param} [Mpps]",
- f"{item[u'title']} Stdev [Mpps]"
- ]
- )
- header.extend(
- [
- f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'reference'][u'title']} Stdev [Mpps]",
- f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'compare'][u'title']} Stdev [Mpps]",
- u"Delta [%]"
- ]
- )
- header_str = u",".join(header) + u"\n"
- except (AttributeError, KeyError) as err:
- logging.error(f"The model is invalid, missing parameter: {repr(err)}")
- return
-
- # Prepare data to the table:
- tbl_dict = dict()
- # topo = ""
- for job, builds in table[u"reference"][u"data"].items():
- # topo = u"2n-skx" if u"2n-skx" in job else u""
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- groups = re.search(REGEX_NIC, tst_data[u"parent"])
- nic = groups.group(0) if groups else u""
- name = \
- f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- _tpc_insert_data(target=tbl_dict[tst_name_mod][u"ref-data"],
- src=tst_data,
- include_tests=table[u"include-tests"])
-
- replacement = table[u"reference"].get(u"data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = \
- f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod][u"ref-data"] = list()
-
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"ref-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- for job, builds in table[u"compare"][u"data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- groups = re.search(REGEX_NIC, tst_data[u"parent"])
- nic = groups.group(0) if groups else u""
- name = \
- f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"cmp-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- replacement = table[u"compare"].get(u"data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = \
- f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod][u"cmp-data"] = list()
-
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"cmp-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- for item in history:
- for job, builds in item[u"data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- continue
- if tbl_dict[tst_name_mod].get(u"history", None) is None:
- tbl_dict[tst_name_mod][u"history"] = OrderedDict()
- if tbl_dict[tst_name_mod][u"history"].\
- get(item[u"title"], None) is None:
- tbl_dict[tst_name_mod][u"history"][item[
- u"title"]] = list()
- try:
- if table[u"include-tests"] == u"MRR":
- res = tst_data[u"result"][u"receive-rate"]
- elif table[u"include-tests"] == u"PDR":
- res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
- elif table[u"include-tests"] == u"NDR":
- res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
- else:
- continue
- tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
- append(res)
- except (TypeError, KeyError):
- pass
-
- tbl_lst = list()
- footnote = False
- for tst_name in tbl_dict:
- item = [tbl_dict[tst_name][u"name"], ]
- if history:
- if tbl_dict[tst_name].get(u"history", None) is not None:
- for hist_data in tbl_dict[tst_name][u"history"].values():
- if hist_data:
- item.append(round(mean(hist_data) / 1000000, 2))
- item.append(round(stdev(hist_data) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- else:
- item.extend([u"Not tested", u"Not tested"])
- data_t = tbl_dict[tst_name][u"ref-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- data_t = tbl_dict[tst_name][u"cmp-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- if item[-2] == u"Not tested":
- pass
- elif item[-4] == u"Not tested":
- item.append(u"New in CSIT-2001")
- # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
- # item.append(u"See footnote [1]")
- # footnote = True
- elif item[-4] != 0:
- item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- if (len(item) == len(header)) and (item[-3] != u"Not tested"):
- tbl_lst.append(item)
-
- tbl_lst = _tpc_sort_table(tbl_lst)
-
- # Generate csv tables:
- csv_file = f"{table[u'output-file']}.csv"
- with open(csv_file, u"wt") as file_handler:
- file_handler.write(header_str)
- for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
-
- txt_file_name = f"{table[u'output-file']}.txt"
- convert_csv_to_pretty_txt(csv_file, txt_file_name)
-
- if footnote:
- with open(txt_file_name, u'a') as txt_file:
- txt_file.writelines([
- u"\nFootnotes:\n",
- u"[1] CSIT-1908 changed test methodology of dot1q tests in "
- u"2-node testbeds, dot1q encapsulation is now used on both "
- u"links of SUT.\n",
- u" Previously dot1q was used only on a single link with the "
- u"other link carrying untagged Ethernet frames. This changes "
- u"results\n",
- u" in slightly lower throughput in CSIT-1908 for these "
- u"tests. See release notes."
- ])
-
- # Generate html table:
- _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
-
-
-def table_perf_comparison_nic(table, input_data):
- """Generate the table(s) with algorithm: table_perf_comparison
- specified in the specification file.
-
- :param table: Table to generate.
- :param input_data: Data to process.
- :type table: pandas.Series
- :type input_data: InputData
- """
-
- logging.info(f" Generating the table {table.get(u'title', u'')} ...")
-
- # Transform the data
- logging.info(
- f" Creating the data set for the {table.get(u'type', u'')} "
- f"{table.get(u'title', u'')}."
- )
- data = input_data.filter_data(table, continue_on_error=True)
-
- # Prepare the header of the tables
- try:
- header = [u"Test case", ]
-
- if table[u"include-tests"] == u"MRR":
- hdr_param = u"Rec Rate"
- else:
- hdr_param = u"Thput"
-
- history = table.get(u"history", list())
- for item in history:
- header.extend(
- [
- f"{item[u'title']} {hdr_param} [Mpps]",
- f"{item[u'title']} Stdev [Mpps]"
- ]
- )
- header.extend(
- [
- f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'reference'][u'title']} Stdev [Mpps]",
- f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'compare'][u'title']} Stdev [Mpps]",
- u"Delta [%]"
- ]
- )
- header_str = u",".join(header) + u"\n"
- except (AttributeError, KeyError) as err:
- logging.error(f"The model is invalid, missing parameter: {repr(err)}")
- return
-
- # Prepare data to the table:
- tbl_dict = dict()
- # topo = u""
- for job, builds in table[u"reference"][u"data"].items():
- # topo = u"2n-skx" if u"2n-skx" in job else u""
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"ref-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- replacement = table[u"reference"].get(u"data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].items():
- if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = \
- f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod][u"ref-data"] = list()
-
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"ref-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- for job, builds in table[u"compare"][u"data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"cmp-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- replacement = table[u"compare"].get(u"data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].items():
- if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = \
- f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod][u"cmp-data"] = list()
-
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"cmp-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- for item in history:
- for job, builds in item[u"data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- if item[u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- continue
- if tbl_dict[tst_name_mod].get(u"history", None) is None:
- tbl_dict[tst_name_mod][u"history"] = OrderedDict()
- if tbl_dict[tst_name_mod][u"history"].\
- get(item[u"title"], None) is None:
- tbl_dict[tst_name_mod][u"history"][item[
- u"title"]] = list()
- try:
- if table[u"include-tests"] == u"MRR":
- res = tst_data[u"result"][u"receive-rate"]
- elif table[u"include-tests"] == u"PDR":
- res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
- elif table[u"include-tests"] == u"NDR":
- res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
- else:
- continue
- tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
- append(res)
- except (TypeError, KeyError):
- pass
-
- tbl_lst = list()
- footnote = False
- for tst_name in tbl_dict:
- item = [tbl_dict[tst_name][u"name"], ]
- if history:
- if tbl_dict[tst_name].get(u"history", None) is not None:
- for hist_data in tbl_dict[tst_name][u"history"].values():
- if hist_data:
- item.append(round(mean(hist_data) / 1000000, 2))
- item.append(round(stdev(hist_data) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- else:
- item.extend([u"Not tested", u"Not tested"])
- data_t = tbl_dict[tst_name][u"ref-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- data_t = tbl_dict[tst_name][u"cmp-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- if item[-2] == u"Not tested":
- pass
- elif item[-4] == u"Not tested":
- item.append(u"New in CSIT-2001")
- # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
- # item.append(u"See footnote [1]")
- # footnote = True
- elif item[-4] != 0:
- item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- if (len(item) == len(header)) and (item[-3] != u"Not tested"):
- tbl_lst.append(item)
-
- tbl_lst = _tpc_sort_table(tbl_lst)
-
- # Generate csv tables:
- csv_file = f"{table[u'output-file']}.csv"
- with open(csv_file, u"wt") as file_handler:
- file_handler.write(header_str)
- for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
-
- txt_file_name = f"{table[u'output-file']}.txt"
- convert_csv_to_pretty_txt(csv_file, txt_file_name)
-
- if footnote:
- with open(txt_file_name, u'a') as txt_file:
- txt_file.writelines([
- u"\nFootnotes:\n",
- u"[1] CSIT-1908 changed test methodology of dot1q tests in "
- u"2-node testbeds, dot1q encapsulation is now used on both "
- u"links of SUT.\n",
- u" Previously dot1q was used only on a single link with the "
- u"other link carrying untagged Ethernet frames. This changes "
- u"results\n",
- u" in slightly lower throughput in CSIT-1908 for these "
- u"tests. See release notes."
- ])
-
- # Generate html table:
- _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
-
-
-def table_nics_comparison(table, input_data):
- """Generate the table(s) with algorithm: table_nics_comparison
- specified in the specification file.
-
- :param table: Table to generate.
- :param input_data: Data to process.
- :type table: pandas.Series
- :type input_data: InputData
- """
-
- logging.info(f" Generating the table {table.get(u'title', u'')} ...")
-
- # Transform the data
- logging.info(
- f" Creating the data set for the {table.get(u'type', u'')} "
- f"{table.get(u'title', u'')}."
- )
- data = input_data.filter_data(table, continue_on_error=True)
+ length. The length of these lists must be the same as the length of the
+ header.
+ :param out_file_name: The name (relative or full path) where the
+ generated html table is written.
+ :param legend: The legend to display below the table.
+ :param footnote: The footnote to display below the table (and legend).
+ :param sort_data: If True the data sorting is enabled.
+ :param title: The table (and file) title.
+ :param generate_rst: If True, wrapping rst file is generated.
+ :type header: list
+ :type data: list of lists
+ :type out_file_name: str
+ :type legend: str
+ :type footnote: str
+ :type sort_data: bool
+ :type title: str
+ :type generate_rst: bool
+ """
- # Prepare the header of the tables
try:
- header = [u"Test case", ]
+ idx = header.index(u"Test Case")
+ except ValueError:
+ idx = 0
+ params = {
+ u"align-hdr": (
+ [u"left", u"right"],
+ [u"left", u"left", u"right"],
+ [u"left", u"left", u"left", u"right"]
+ ),
+ u"align-itm": (
+ [u"left", u"right"],
+ [u"left", u"left", u"right"],
+ [u"left", u"left", u"left", u"right"]
+ ),
+ u"width": ([15, 9], [4, 24, 10], [4, 4, 32, 10])
+ }
- if table[u"include-tests"] == u"MRR":
- hdr_param = u"Rec Rate"
- else:
- hdr_param = u"Thput"
+ df_data = pd.DataFrame(data, columns=header)
- header.extend(
- [
- f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'reference'][u'title']} Stdev [Mpps]",
- f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'compare'][u'title']} Stdev [Mpps]",
- u"Delta [%]"
- ]
+ if sort_data:
+ df_sorted = [df_data.sort_values(
+ by=[key, header[idx]], ascending=[True, True]
+ if key != header[idx] else [False, True]) for key in header]
+ df_sorted_rev = [df_data.sort_values(
+ by=[key, header[idx]], ascending=[False, True]
+ if key != header[idx] else [True, True]) for key in header]
+ df_sorted.extend(df_sorted_rev)
+ else:
+ df_sorted = df_data
+
+ fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
+ for idx in range(len(df_data))]]
+ table_header = dict(
+ values=[f"{item.replace(u',', u',
')}" for item in header],
+ fill_color=u"#7eade7",
+ align=params[u"align-hdr"][idx],
+ font=dict(
+ family=u"Courier New",
+ size=12
)
+ )
- except (AttributeError, KeyError) as err:
- logging.error(f"The model is invalid, missing parameter: {repr(err)}")
- return
+ fig = go.Figure()
- # Prepare data to the table:
- tbl_dict = dict()
- for job, builds in table[u"data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if tbl_dict.get(tst_name_mod, None) is None:
- name = u"-".join(tst_data[u"name"].split(u"-")[:-1])
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- try:
- result = None
- if table[u"include-tests"] == u"MRR":
- result = tst_data[u"result"][u"receive-rate"]
- elif table[u"include-tests"] == u"PDR":
- result = tst_data[u"throughput"][u"PDR"][u"LOWER"]
- elif table[u"include-tests"] == u"NDR":
- result = tst_data[u"throughput"][u"NDR"][u"LOWER"]
- else:
- continue
+ if sort_data:
+ for table in df_sorted:
+ columns = [table.get(col) for col in header]
+ fig.add_trace(
+ go.Table(
+ columnwidth=params[u"width"][idx],
+ header=table_header,
+ cells=dict(
+ values=columns,
+ fill_color=fill_color,
+ align=params[u"align-itm"][idx],
+ font=dict(
+ family=u"Courier New",
+ size=12
+ )
+ )
+ )
+ )
- if result and \
- table[u"reference"][u"nic"] in tst_data[u"tags"]:
- tbl_dict[tst_name_mod][u"ref-data"].append(result)
- elif result and \
- table[u"compare"][u"nic"] in tst_data[u"tags"]:
- tbl_dict[tst_name_mod][u"cmp-data"].append(result)
- except (TypeError, KeyError) as err:
- logging.debug(f"No data for {tst_name}\n{repr(err)}")
- # No data in output.xml for this test
+ buttons = list()
+ menu_items = [f"{itm} (ascending)" for itm in header]
+ menu_items.extend([f"{itm} (descending)" for itm in header])
+ for idx, hdr in enumerate(menu_items):
+ visible = [False, ] * len(menu_items)
+ visible[idx] = True
+ buttons.append(
+ dict(
+ label=hdr.replace(u" [Mpps]", u""),
+ method=u"update",
+ args=[{u"visible": visible}],
+ )
+ )
- tbl_lst = list()
- for tst_name in tbl_dict:
- item = [tbl_dict[tst_name][u"name"], ]
- data_t = tbl_dict[tst_name][u"ref-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend([None, None])
- data_t = tbl_dict[tst_name][u"cmp-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend([None, None])
- if item[-4] is not None and item[-2] is not None and item[-4] != 0:
- item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- if len(item) == len(header):
- tbl_lst.append(item)
+ fig.update_layout(
+ updatemenus=[
+ go.layout.Updatemenu(
+ type=u"dropdown",
+ direction=u"down",
+ x=0.0,
+ xanchor=u"left",
+ y=1.002,
+ yanchor=u"bottom",
+ active=len(menu_items) - 1,
+ buttons=list(buttons)
+ )
+ ],
+ )
+ else:
+ fig.add_trace(
+ go.Table(
+ columnwidth=params[u"width"][idx],
+ header=table_header,
+ cells=dict(
+ values=[df_sorted.get(col) for col in header],
+ fill_color=fill_color,
+ align=params[u"align-itm"][idx],
+ font=dict(
+ family=u"Courier New",
+ size=12
+ )
+ )
+ )
+ )
- # Sort the table according to the relative change
- tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
+ ploff.plot(
+ fig,
+ show_link=False,
+ auto_open=False,
+ filename=f"{out_file_name}_in.html"
+ )
- # Generate csv tables:
- with open(f"{table[u'output-file']}.csv", u"wt") as file_handler:
- file_handler.write(u",".join(header) + u"\n")
- for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
+ if not generate_rst:
+ return
- convert_csv_to_pretty_txt(f"{table[u'output-file']}.csv",
- f"{table[u'output-file']}.txt")
+ file_name = out_file_name.split(u"/")[-1]
+ if u"vpp" in out_file_name:
+ path = u"_tmp/src/vpp_performance_tests/comparisons/"
+ else:
+ path = u"_tmp/src/dpdk_performance_tests/comparisons/"
+ logging.info(f" Writing the HTML file to {path}{file_name}.rst")
+ with open(f"{path}{file_name}.rst", u"wt") as rst_file:
+ rst_file.write(
+ u"\n"
+ u".. |br| raw:: html\n\n
\n\n\n"
+ u".. |prein| raw:: html\n\n \n\n\n"
+ u".. |preout| raw:: html\n\n
\n\n"
+ )
+ if title:
+ rst_file.write(f"{title}\n")
+ rst_file.write(f"{u'`' * len(title)}\n\n")
+ rst_file.write(
+ u".. raw:: html\n\n"
+ f' \n\n'
+ )
- # Generate html table:
- _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
+ if legend:
+ try:
+ itm_lst = legend[1:-2].split(u"\n")
+ rst_file.write(
+ f"{itm_lst[0]}\n\n- " + u'\n- '.join(itm_lst[1:]) + u"\n\n"
+ )
+ except IndexError as err:
+ logging.error(f"Legend cannot be written to html file\n{err}")
+ if footnote:
+ try:
+ itm_lst = footnote[1:].split(u"\n")
+ rst_file.write(
+ f"{itm_lst[0]}\n\n- " + u'\n- '.join(itm_lst[1:]) + u"\n\n"
+ )
+ except IndexError as err:
+ logging.error(f"Footnote cannot be written to html file\n{err}")
def table_soak_vs_ndr(table, input_data):
@@ -1083,14 +691,36 @@ def table_soak_vs_ndr(table, input_data):
# Prepare the header of the table
try:
header = [
- u"Test case",
- f"{table[u'reference'][u'title']} Thput [Mpps]",
- f"{table[u'reference'][u'title']} Stdev [Mpps]",
- f"{table[u'compare'][u'title']} Thput [Mpps]",
- f"{table[u'compare'][u'title']} Stdev [Mpps]",
- u"Delta [%]", u"Stdev of delta [%]"
+ u"Test Case",
+ f"Avg({table[u'reference'][u'title']})",
+ f"Stdev({table[u'reference'][u'title']})",
+ f"Avg({table[u'compare'][u'title']})",
+ f"Stdev{table[u'compare'][u'title']})",
+ u"Diff",
+ u"Stdev(Diff)"
]
- header_str = u",".join(header) + u"\n"
+ header_str = u";".join(header) + u"\n"
+ legend = (
+ u"\nLegend:\n"
+ f"Avg({table[u'reference'][u'title']}): "
+ f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
+ f"from a series of runs of the listed tests.\n"
+ f"Stdev({table[u'reference'][u'title']}): "
+ f"Standard deviation value of {table[u'reference'][u'title']} "
+ f"[Mpps] computed from a series of runs of the listed tests.\n"
+ f"Avg({table[u'compare'][u'title']}): "
+ f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
+ f"a series of runs of the listed tests.\n"
+ f"Stdev({table[u'compare'][u'title']}): "
+ f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
+ f"computed from a series of runs of the listed tests.\n"
+ f"Diff({table[u'reference'][u'title']},"
+ f"{table[u'compare'][u'title']}): "
+ f"Percentage change calculated for mean values.\n"
+ u"Stdev(Diff): "
+ u"Standard deviation of percentage change calculated for mean "
+ u"values."
+ )
except (AttributeError, KeyError) as err:
logging.error(f"The model is invalid, missing parameter: {repr(err)}")
return
@@ -1133,7 +763,8 @@ def table_soak_vs_ndr(table, input_data):
if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
continue
if table[u"include-tests"] == u"MRR":
- result = tst_data[u"result"][u"receive-rate"]
+ result = (tst_data[u"result"][u"receive-rate"],
+ tst_data[u"result"][u"receive-stdev"])
elif table[u"include-tests"] == u"PDR":
result = \
tst_data[u"throughput"][u"PDR"][u"LOWER"]
@@ -1153,45 +784,69 @@ def table_soak_vs_ndr(table, input_data):
item = [tbl_dict[tst_name][u"name"], ]
data_r = tbl_dict[tst_name][u"ref-data"]
if data_r:
- data_r_mean = mean(data_r)
- item.append(round(data_r_mean / 1000000, 2))
- data_r_stdev = stdev(data_r)
- item.append(round(data_r_stdev / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ data_r_mean = data_r[0][0]
+ data_r_stdev = data_r[0][1]
+ else:
+ data_r_mean = mean(data_r)
+ data_r_stdev = stdev(data_r)
+ item.append(round(data_r_mean / 1e6, 1))
+ item.append(round(data_r_stdev / 1e6, 1))
else:
data_r_mean = None
data_r_stdev = None
item.extend([None, None])
data_c = tbl_dict[tst_name][u"cmp-data"]
if data_c:
- data_c_mean = mean(data_c)
- item.append(round(data_c_mean / 1000000, 2))
- data_c_stdev = stdev(data_c)
- item.append(round(data_c_stdev / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ data_c_mean = data_c[0][0]
+ data_c_stdev = data_c[0][1]
+ else:
+ data_c_mean = mean(data_c)
+ data_c_stdev = stdev(data_c)
+ item.append(round(data_c_mean / 1e6, 1))
+ item.append(round(data_c_stdev / 1e6, 1))
else:
data_c_mean = None
data_c_stdev = None
item.extend([None, None])
- if data_r_mean and data_c_mean:
+ if data_r_mean is not None and data_c_mean is not None:
delta, d_stdev = relative_change_stdev(
data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
- item.append(round(delta, 2))
- item.append(round(d_stdev, 2))
+ try:
+ item.append(round(delta))
+ except ValueError:
+ item.append(delta)
+ try:
+ item.append(round(d_stdev))
+ except ValueError:
+ item.append(d_stdev)
tbl_lst.append(item)
# Sort the table according to the relative change
tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
# Generate csv tables:
- csv_file = f"{table[u'output-file']}.csv"
- with open(csv_file, u"wt") as file_handler:
+ csv_file_name = f"{table[u'output-file']}.csv"
+ with open(csv_file_name, u"wt") as file_handler:
file_handler.write(header_str)
for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
+ file_handler.write(u";".join([str(item) for item in test]) + u"\n")
- convert_csv_to_pretty_txt(csv_file, f"{table[u'output-file']}.txt")
+ convert_csv_to_pretty_txt(
+ csv_file_name, f"{table[u'output-file']}.txt", delimiter=u";"
+ )
+ with open(f"{table[u'output-file']}.txt", u'a') as file_handler:
+ file_handler.write(legend)
# Generate html table:
- _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
+ _tpc_generate_html_table(
+ header,
+ tbl_lst,
+ table[u'output-file'],
+ legend=legend,
+ title=table.get(u"title", u"")
+ )
def table_perf_trending_dash(table, input_data):
@@ -1225,6 +880,8 @@ def table_perf_trending_dash(table, input_data):
]
header_str = u",".join(header) + u"\n"
+ incl_tests = table.get(u"include-tests", u"MRR")
+
# Prepare data to the table:
tbl_dict = dict()
for job, builds in table[u"data"].items():
@@ -1242,8 +899,15 @@ def table_perf_trending_dash(table, input_data):
u"data": OrderedDict()
}
try:
- tbl_dict[tst_name][u"data"][str(build)] = \
- tst_data[u"result"][u"receive-rate"]
+ if incl_tests == u"MRR":
+ tbl_dict[tst_name][u"data"][str(build)] = \
+ tst_data[u"result"][u"receive-rate"]
+ elif incl_tests == u"NDR":
+ tbl_dict[tst_name][u"data"][str(build)] = \
+ tst_data[u"throughput"][u"NDR"][u"LOWER"]
+ elif incl_tests == u"PDR":
+ tbl_dict[tst_name][u"data"][str(build)] = \
+ tst_data[u"throughput"][u"PDR"][u"LOWER"]
except (TypeError, KeyError):
pass # No data in output.xml for this test
@@ -1253,7 +917,11 @@ def table_perf_trending_dash(table, input_data):
if len(data_t) < 2:
continue
- classification_lst, avgs = classify_anomalies(data_t)
+ try:
+ classification_lst, avgs, _ = classify_anomalies(data_t)
+ except ValueError as err:
+ logging.info(f"{err} Skipping")
+ return
win_size = min(len(data_t), table[u"window"])
long_win_size = min(len(data_t), table[u"long-trend-window"])
@@ -1271,13 +939,13 @@ def table_perf_trending_dash(table, input_data):
rel_change_last = nan
else:
rel_change_last = round(
- ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
+ ((last_avg - avg_week_ago) / avg_week_ago) * 1e2, 2)
if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
rel_change_long = nan
else:
rel_change_long = round(
- ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
+ ((last_avg - max_long_avg) / max_long_avg) * 1e2, 2)
if classification_lst:
if isnan(rel_change_last) and isnan(rel_change_long):
@@ -1287,20 +955,21 @@ def table_perf_trending_dash(table, input_data):
continue
tbl_lst.append(
[tbl_dict[tst_name][u"name"],
- round(last_avg / 1000000, 2),
+ round(last_avg / 1e6, 2),
rel_change_last,
rel_change_long,
- classification_lst[-win_size:].count(u"regression"),
- classification_lst[-win_size:].count(u"progression")])
+ classification_lst[-win_size+1:].count(u"regression"),
+ classification_lst[-win_size+1:].count(u"progression")])
tbl_lst.sort(key=lambda rel: rel[0])
+ tbl_lst.sort(key=lambda rel: rel[3])
+ tbl_lst.sort(key=lambda rel: rel[2])
tbl_sorted = list()
for nrr in range(table[u"window"], -1, -1):
tbl_reg = [item for item in tbl_lst if item[4] == nrr]
for nrp in range(table[u"window"], -1, -1):
tbl_out = [item for item in tbl_reg if item[5] == nrp]
- tbl_out.sort(key=lambda rel: rel[2])
tbl_sorted.extend(tbl_out)
file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
@@ -1363,27 +1032,27 @@ def _generate_url(testbed, test_name):
if u"1t1c" in test_name or \
(u"-1c-" in test_name and
- testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
+ testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv", u"2n-tx2")):
cores = u"1t1c"
elif u"2t2c" in test_name or \
(u"-2c-" in test_name and
- testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
+ testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv", u"2n-tx2")):
cores = u"2t2c"
elif u"4t4c" in test_name or \
(u"-4c-" in test_name and
- testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
+ testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv", u"2n-tx2")):
cores = u"4t4c"
elif u"2t1c" in test_name or \
(u"-1c-" in test_name and
- testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
+ testbed in (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2")):
cores = u"2t1c"
elif u"4t2c" in test_name or \
(u"-2c-" in test_name and
- testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
+ testbed in (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2")):
cores = u"4t2c"
elif u"8t4c" in test_name or \
(u"-4c-" in test_name and
- testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
+ testbed in (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2")):
cores = u"8t4c"
else:
cores = u""
@@ -1399,13 +1068,55 @@ def _generate_url(testbed, test_name):
elif u"dnv" in testbed or u"tsh" in testbed:
driver = u"ixgbe"
else:
- driver = u"i40e"
-
- if u"acl" in test_name or \
- u"macip" in test_name or \
- u"nat" in test_name or \
- u"policer" in test_name or \
- u"cop" in test_name:
+ driver = u"dpdk"
+
+ if u"macip-iacl1s" in test_name:
+ bsf = u"features-macip-iacl1"
+ elif u"macip-iacl10s" in test_name:
+ bsf = u"features-macip-iacl10"
+ elif u"macip-iacl50s" in test_name:
+ bsf = u"features-macip-iacl50"
+ elif u"iacl1s" in test_name:
+ bsf = u"features-iacl1"
+ elif u"iacl10s" in test_name:
+ bsf = u"features-iacl10"
+ elif u"iacl50s" in test_name:
+ bsf = u"features-iacl50"
+ elif u"oacl1s" in test_name:
+ bsf = u"features-oacl1"
+ elif u"oacl10s" in test_name:
+ bsf = u"features-oacl10"
+ elif u"oacl50s" in test_name:
+ bsf = u"features-oacl50"
+ elif u"nat44det" in test_name:
+ bsf = u"nat44det-bidir"
+ elif u"nat44ed" in test_name and u"udir" in test_name:
+ bsf = u"nat44ed-udir"
+ elif u"-cps" in test_name and u"ethip4udp" in test_name:
+ bsf = u"udp-cps"
+ elif u"-cps" in test_name and u"ethip4tcp" in test_name:
+ bsf = u"tcp-cps"
+ elif u"-pps" in test_name and u"ethip4udp" in test_name:
+ bsf = u"udp-pps"
+ elif u"-pps" in test_name and u"ethip4tcp" in test_name:
+ bsf = u"tcp-pps"
+ elif u"-tput" in test_name and u"ethip4udp" in test_name:
+ bsf = u"udp-tput"
+ elif u"-tput" in test_name and u"ethip4tcp" in test_name:
+ bsf = u"tcp-tput"
+ elif u"udpsrcscale" in test_name:
+ bsf = u"features-udp"
+ elif u"iacl" in test_name:
+ bsf = u"features"
+ elif u"policer" in test_name:
+ bsf = u"features"
+ elif u"adl" in test_name:
+ bsf = u"features"
+ elif u"cop" in test_name:
+ bsf = u"features"
+ elif u"nat" in test_name:
+ bsf = u"features"
+ elif u"macip" in test_name:
bsf = u"features"
elif u"scale" in test_name:
bsf = u"scale"
@@ -1416,6 +1127,24 @@ def _generate_url(testbed, test_name):
if u"114b" in test_name and u"vhost" in test_name:
domain = u"vts"
+ elif u"nat44" in test_name or u"-pps" in test_name or u"-cps" in test_name:
+ domain = u"nat44"
+ if u"nat44det" in test_name:
+ domain += u"-det-bidir"
+ else:
+ domain += u"-ed"
+ if u"udir" in test_name:
+ domain += u"-unidir"
+ elif u"-ethip4udp-" in test_name:
+ domain += u"-udp"
+ elif u"-ethip4tcp-" in test_name:
+ domain += u"-tcp"
+ if u"-cps" in test_name:
+ domain += u"-cps"
+ elif u"-pps" in test_name:
+ domain += u"-pps"
+ elif u"-tput" in test_name:
+ domain += u"-tput"
elif u"testpmd" in test_name or u"l3fwd" in test_name:
domain = u"dpdk"
elif u"memif" in test_name:
@@ -1444,6 +1173,8 @@ def _generate_url(testbed, test_name):
bsf += u"-hw"
elif u"ethip4vxlan" in test_name:
domain = u"ip4_tunnels"
+ elif u"ethip4udpgeneve" in test_name:
+ domain = u"ip4_tunnels"
elif u"ip4base" in test_name or u"ip4scale" in test_name:
domain = u"ip4"
elif u"ip6base" in test_name or u"ip6scale" in test_name:
@@ -1479,15 +1210,33 @@ def table_perf_trending_dash_html(table, input_data):
if not table.get(u"testbed", None):
logging.error(
f"The testbed is not defined for the table "
- f"{table.get(u'title', u'')}."
+ f"{table.get(u'title', u'')}. Skipping."
+ )
+ return
+
+ test_type = table.get(u"test-type", u"MRR")
+ if test_type not in (u"MRR", u"NDR", u"PDR"):
+ logging.error(
+ f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
+ f"Skipping."
)
return
+ if test_type in (u"NDR", u"PDR"):
+ lnk_dir = u"../ndrpdr_trending/"
+ lnk_sufix = f"-{test_type.lower()}"
+ else:
+ lnk_dir = u"../trending/"
+ lnk_sufix = u""
+
logging.info(f" Generating the table {table.get(u'title', u'')} ...")
try:
with open(table[u"input-file"], u'rt') as csv_file:
csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
+ except FileNotFoundError as err:
+ logging.warning(f"{err}")
+ return
except KeyError:
logging.warning(u"The input file is not defined.")
return
@@ -1542,13 +1291,14 @@ def table_perf_trending_dash_html(table, input_data):
attrib=dict(align=u"left" if c_idx == 0 else u"center")
)
# Name:
- if c_idx == 0:
+ if c_idx == 0 and table.get(u"add-links", True):
ref = ET.SubElement(
tdata,
u"a",
attrib=dict(
- href=f"../trending/"
- f"{_generate_url(table.get(u'testbed', ''), item)}"
+ href=f"{lnk_dir}"
+ f"{_generate_url(table.get(u'testbed', ''), item)}"
+ f"{lnk_sufix}"
)
)
ref.text = item
@@ -1598,6 +1348,8 @@ def table_last_failed_tests(table, input_data):
build = str(build)
try:
version = input_data.metadata(job, build).get(u"version", u"")
+ duration = \
+ input_data.metadata(job, build).get(u"elapsedtime", u"")
except KeyError:
logging.error(f"Data for {job}: {build} is not present.")
return
@@ -1616,15 +1368,16 @@ def table_last_failed_tests(table, input_data):
continue
nic = groups.group(0)
failed_tests.append(f"{nic}-{tst_data[u'name']}")
- tbl_list.append(str(passed))
- tbl_list.append(str(failed))
+ tbl_list.append(passed)
+ tbl_list.append(failed)
+ tbl_list.append(duration)
tbl_list.extend(failed_tests)
file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
logging.info(f" Writing file: {file_name}")
with open(file_name, u"wt") as file_handler:
for test in tbl_list:
- file_handler.write(test + u'\n')
+ file_handler.write(f"{test}\n")
def table_failed_tests(table, input_data):
@@ -1646,6 +1399,10 @@ def table_failed_tests(table, input_data):
)
data = input_data.filter_data(table, continue_on_error=True)
+ test_type = u"MRR"
+ if u"NDRPDR" in table.get(u"filter", list()):
+ test_type = u"NDRPDR"
+
# Prepare the header of the tables
header = [
u"Test Case",
@@ -1709,15 +1466,14 @@ def table_failed_tests(table, input_data):
fails_last_csit = val[3]
if fails_nr:
max_fails = fails_nr if fails_nr > max_fails else max_fails
- tbl_lst.append(
- [
- tst_data[u"name"],
- fails_nr,
- fails_last_date,
- fails_last_vpp,
- f"mrr-daily-build-{fails_last_csit}"
- ]
- )
+ tbl_lst.append([
+ tst_data[u"name"],
+ fails_nr,
+ fails_last_date,
+ fails_last_vpp,
+ f"{u'mrr-daily' if test_type == u'MRR' else u'ndrpdr-weekly'}"
+ f"-build-{fails_last_csit}"
+ ])
tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
tbl_sorted = list()
@@ -1751,10 +1507,25 @@ def table_failed_tests_html(table, input_data):
if not table.get(u"testbed", None):
logging.error(
f"The testbed is not defined for the table "
- f"{table.get(u'title', u'')}."
+ f"{table.get(u'title', u'')}. Skipping."
+ )
+ return
+
+ test_type = table.get(u"test-type", u"MRR")
+ if test_type not in (u"MRR", u"NDR", u"PDR", u"NDRPDR"):
+ logging.error(
+ f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
+ f"Skipping."
)
return
+ if test_type in (u"NDRPDR", u"NDR", u"PDR"):
+ lnk_dir = u"../ndrpdr_trending/"
+ lnk_sufix = u"-pdr"
+ else:
+ lnk_dir = u"../trending/"
+ lnk_sufix = u""
+
logging.info(f" Generating the table {table.get(u'title', u'')} ...")
try:
@@ -1796,13 +1567,14 @@ def table_failed_tests_html(table, input_data):
attrib=dict(align=u"left" if c_idx == 0 else u"center")
)
# Name:
- if c_idx == 0:
+ if c_idx == 0 and table.get(u"add-links", True):
ref = ET.SubElement(
tdata,
u"a",
attrib=dict(
- href=f"../trending/"
- f"{_generate_url(table.get(u'testbed', ''), item)}"
+ href=f"{lnk_dir}"
+ f"{_generate_url(table.get(u'testbed', ''), item)}"
+ f"{lnk_sufix}"
)
)
ref.text = item
@@ -1817,3 +1589,546 @@ def table_failed_tests_html(table, input_data):
except KeyError:
logging.warning(u"The output file is not defined.")
return
+
+
+def table_comparison(table, input_data):
+ """Generate the table(s) with algorithm: table_comparison
+ specified in the specification file.
+
+ :param table: Table to generate.
+ :param input_data: Data to process.
+ :type table: pandas.Series
+ :type input_data: InputData
+ """
+ logging.info(f" Generating the table {table.get(u'title', u'')} ...")
+
+ # Transform the data
+ logging.info(
+ f" Creating the data set for the {table.get(u'type', u'')} "
+ f"{table.get(u'title', u'')}."
+ )
+
+ columns = table.get(u"columns", None)
+ if not columns:
+ logging.error(
+ f"No columns specified for {table.get(u'title', u'')}. Skipping."
+ )
+ return
+
+ cols = list()
+ for idx, col in enumerate(columns):
+ if col.get(u"data-set", None) is None:
+ logging.warning(f"No data for column {col.get(u'title', u'')}")
+ continue
+ tag = col.get(u"tag", None)
+ data = input_data.filter_data(
+ table,
+ params=[u"throughput", u"result", u"name", u"parent", u"tags"],
+ data=col[u"data-set"],
+ continue_on_error=True
+ )
+ col_data = {
+ u"title": col.get(u"title", f"Column{idx}"),
+ u"data": dict()
+ }
+ for builds in data.values:
+ for build in builds:
+ for tst_name, tst_data in build.items():
+ if tag and tag not in tst_data[u"tags"]:
+ continue
+ tst_name_mod = \
+ _tpc_modify_test_name(tst_name, ignore_nic=True).\
+ replace(u"2n1l-", u"")
+ if col_data[u"data"].get(tst_name_mod, None) is None:
+ name = tst_data[u'name'].rsplit(u'-', 1)[0]
+ if u"across testbeds" in table[u"title"].lower() or \
+ u"across topologies" in table[u"title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ col_data[u"data"][tst_name_mod] = {
+ u"name": name,
+ u"replace": True,
+ u"data": list(),
+ u"mean": None,
+ u"stdev": None
+ }
+ _tpc_insert_data(
+ target=col_data[u"data"][tst_name_mod],
+ src=tst_data,
+ include_tests=table[u"include-tests"]
+ )
+
+ replacement = col.get(u"data-replacement", None)
+ if replacement:
+ rpl_data = input_data.filter_data(
+ table,
+ params=[u"throughput", u"result", u"name", u"parent", u"tags"],
+ data=replacement,
+ continue_on_error=True
+ )
+ for builds in rpl_data.values:
+ for build in builds:
+ for tst_name, tst_data in build.items():
+ if tag and tag not in tst_data[u"tags"]:
+ continue
+ tst_name_mod = \
+ _tpc_modify_test_name(tst_name, ignore_nic=True).\
+ replace(u"2n1l-", u"")
+ if col_data[u"data"].get(tst_name_mod, None) is None:
+ name = tst_data[u'name'].rsplit(u'-', 1)[0]
+ if u"across testbeds" in table[u"title"].lower() \
+ or u"across topologies" in \
+ table[u"title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ col_data[u"data"][tst_name_mod] = {
+ u"name": name,
+ u"replace": False,
+ u"data": list(),
+ u"mean": None,
+ u"stdev": None
+ }
+ if col_data[u"data"][tst_name_mod][u"replace"]:
+ col_data[u"data"][tst_name_mod][u"replace"] = False
+ col_data[u"data"][tst_name_mod][u"data"] = list()
+ _tpc_insert_data(
+ target=col_data[u"data"][tst_name_mod],
+ src=tst_data,
+ include_tests=table[u"include-tests"]
+ )
+
+ if table[u"include-tests"] in (u"NDR", u"PDR"):
+ for tst_name, tst_data in col_data[u"data"].items():
+ if tst_data[u"data"]:
+ tst_data[u"mean"] = mean(tst_data[u"data"])
+ tst_data[u"stdev"] = stdev(tst_data[u"data"])
+
+ cols.append(col_data)
+
+ tbl_dict = dict()
+ for col in cols:
+ for tst_name, tst_data in col[u"data"].items():
+ if tbl_dict.get(tst_name, None) is None:
+ tbl_dict[tst_name] = {
+ "name": tst_data[u"name"]
+ }
+ tbl_dict[tst_name][col[u"title"]] = {
+ u"mean": tst_data[u"mean"],
+ u"stdev": tst_data[u"stdev"]
+ }
+
+ if not tbl_dict:
+ logging.warning(f"No data for table {table.get(u'title', u'')}!")
+ return
+
+ tbl_lst = list()
+ for tst_data in tbl_dict.values():
+ row = [tst_data[u"name"], ]
+ for col in cols:
+ row.append(tst_data.get(col[u"title"], None))
+ tbl_lst.append(row)
+
+ comparisons = table.get(u"comparisons", None)
+ rcas = list()
+ if comparisons and isinstance(comparisons, list):
+ for idx, comp in enumerate(comparisons):
+ try:
+ col_ref = int(comp[u"reference"])
+ col_cmp = int(comp[u"compare"])
+ except KeyError:
+ logging.warning(u"Comparison: No references defined! Skipping.")
+ comparisons.pop(idx)
+ continue
+ if not (0 < col_ref <= len(cols) and 0 < col_cmp <= len(cols) or
+ col_ref == col_cmp):
+ logging.warning(f"Wrong values of reference={col_ref} "
+ f"and/or compare={col_cmp}. Skipping.")
+ comparisons.pop(idx)
+ continue
+ rca_file_name = comp.get(u"rca-file", None)
+ if rca_file_name:
+ try:
+ with open(rca_file_name, u"r") as file_handler:
+ rcas.append(
+ {
+ u"title": f"RCA{idx + 1}",
+ u"data": load(file_handler, Loader=FullLoader)
+ }
+ )
+ except (YAMLError, IOError) as err:
+ logging.warning(
+ f"The RCA file {rca_file_name} does not exist or "
+ f"it is corrupted!"
+ )
+ logging.debug(repr(err))
+ rcas.append(None)
+ else:
+ rcas.append(None)
+ else:
+ comparisons = None
+
+ tbl_cmp_lst = list()
+ if comparisons:
+ for row in tbl_lst:
+ new_row = deepcopy(row)
+ for comp in comparisons:
+ ref_itm = row[int(comp[u"reference"])]
+ if ref_itm is None and \
+ comp.get(u"reference-alt", None) is not None:
+ ref_itm = row[int(comp[u"reference-alt"])]
+ cmp_itm = row[int(comp[u"compare"])]
+ if ref_itm is not None and cmp_itm is not None and \
+ ref_itm[u"mean"] is not None and \
+ cmp_itm[u"mean"] is not None and \
+ ref_itm[u"stdev"] is not None and \
+ cmp_itm[u"stdev"] is not None:
+ delta, d_stdev = relative_change_stdev(
+ ref_itm[u"mean"], cmp_itm[u"mean"],
+ ref_itm[u"stdev"], cmp_itm[u"stdev"]
+ )
+ if delta is None:
+ break
+ new_row.append({
+ u"mean": delta * 1e6,
+ u"stdev": d_stdev * 1e6
+ })
+ else:
+ break
+ else:
+ tbl_cmp_lst.append(new_row)
+
+ try:
+ tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
+ tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
+ except TypeError as err:
+ logging.warning(f"Empty data element in table\n{tbl_cmp_lst}\n{err}")
+
+ tbl_for_csv = list()
+ for line in tbl_cmp_lst:
+ row = [line[0], ]
+ for idx, itm in enumerate(line[1:]):
+ if itm is None or not isinstance(itm, dict) or\
+ itm.get(u'mean', None) is None or \
+ itm.get(u'stdev', None) is None:
+ row.append(u"NT")
+ row.append(u"NT")
+ else:
+ row.append(round(float(itm[u'mean']) / 1e6, 3))
+ row.append(round(float(itm[u'stdev']) / 1e6, 3))
+ for rca in rcas:
+ if rca is None:
+ continue
+ rca_nr = rca[u"data"].get(row[0], u"-")
+ row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
+ tbl_for_csv.append(row)
+
+ header_csv = [u"Test Case", ]
+ for col in cols:
+ header_csv.append(f"Avg({col[u'title']})")
+ header_csv.append(f"Stdev({col[u'title']})")
+ for comp in comparisons:
+ header_csv.append(
+ f"Avg({comp.get(u'title', u'')})"
+ )
+ header_csv.append(
+ f"Stdev({comp.get(u'title', u'')})"
+ )
+ for rca in rcas:
+ if rca:
+ header_csv.append(rca[u"title"])
+
+ legend_lst = table.get(u"legend", None)
+ if legend_lst is None:
+ legend = u""
+ else:
+ legend = u"\n" + u"\n".join(legend_lst) + u"\n"
+
+ footnote = u""
+ if rcas and any(rcas):
+ footnote += u"\nRoot Cause Analysis:\n"
+ for rca in rcas:
+ if rca:
+ footnote += f"{rca[u'data'].get(u'footnote', u'')}\n"
+
+ csv_file_name = f"{table[u'output-file']}-csv.csv"
+ with open(csv_file_name, u"wt", encoding='utf-8') as file_handler:
+ file_handler.write(
+ u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
+ )
+ for test in tbl_for_csv:
+ file_handler.write(
+ u",".join([f'"{item}"' for item in test]) + u"\n"
+ )
+ if legend_lst:
+ for item in legend_lst:
+ file_handler.write(f'"{item}"\n')
+ if footnote:
+ for itm in footnote.split(u"\n"):
+ file_handler.write(f'"{itm}"\n')
+
+ tbl_tmp = list()
+ max_lens = [0, ] * len(tbl_cmp_lst[0])
+ for line in tbl_cmp_lst:
+ row = [line[0], ]
+ for idx, itm in enumerate(line[1:]):
+ if itm is None or not isinstance(itm, dict) or \
+ itm.get(u'mean', None) is None or \
+ itm.get(u'stdev', None) is None:
+ new_itm = u"NT"
+ else:
+ if idx < len(cols):
+ new_itm = (
+ f"{round(float(itm[u'mean']) / 1e6, 1)} "
+ f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
+ replace(u"nan", u"NaN")
+ )
+ else:
+ new_itm = (
+ f"{round(float(itm[u'mean']) / 1e6, 1):+} "
+ f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
+ replace(u"nan", u"NaN")
+ )
+ if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]:
+ max_lens[idx] = len(new_itm.rsplit(u" ", 1)[-1])
+ row.append(new_itm)
+
+ tbl_tmp.append(row)
+
+ header = [u"Test Case", ]
+ header.extend([col[u"title"] for col in cols])
+ header.extend([comp.get(u"title", u"") for comp in comparisons])
+
+ tbl_final = list()
+ for line in tbl_tmp:
+ row = [line[0], ]
+ for idx, itm in enumerate(line[1:]):
+ if itm in (u"NT", u"NaN"):
+ row.append(itm)
+ continue
+ itm_lst = itm.rsplit(u"\u00B1", 1)
+ itm_lst[-1] = \
+ f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
+ itm_str = u"\u00B1".join(itm_lst)
+
+ if idx >= len(cols):
+ # Diffs
+ rca = rcas[idx - len(cols)]
+ if rca:
+ # Add rcas to diffs
+ rca_nr = rca[u"data"].get(row[0], None)
+ if rca_nr:
+ hdr_len = len(header[idx + 1]) - 1
+ if hdr_len < 19:
+ hdr_len = 19
+ rca_nr = f"[{rca_nr}]"
+ itm_str = (
+ f"{u' ' * (4 - len(rca_nr))}{rca_nr}"
+ f"{u' ' * (hdr_len - 4 - len(itm_str))}"
+ f"{itm_str}"
+ )
+ row.append(itm_str)
+ tbl_final.append(row)
+
+ # Generate csv tables:
+ csv_file_name = f"{table[u'output-file']}.csv"
+ logging.info(f" Writing the file {csv_file_name}")
+ with open(csv_file_name, u"wt", encoding='utf-8') as file_handler:
+ file_handler.write(u";".join(header) + u"\n")
+ for test in tbl_final:
+ file_handler.write(u";".join([str(item) for item in test]) + u"\n")
+
+ # Generate txt table:
+ txt_file_name = f"{table[u'output-file']}.txt"
+ logging.info(f" Writing the file {txt_file_name}")
+ convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u";")
+
+ with open(txt_file_name, u'a', encoding='utf-8') as file_handler:
+ file_handler.write(legend)
+ file_handler.write(footnote)
+
+ # Generate html table:
+ _tpc_generate_html_table(
+ header,
+ tbl_final,
+ table[u'output-file'],
+ legend=legend,
+ footnote=footnote,
+ sort_data=False,
+ title=table.get(u"title", u"")
+ )
+
+
+def table_weekly_comparison(table, in_data):
+ """Generate the table(s) with algorithm: table_weekly_comparison
+ specified in the specification file.
+
+ :param table: Table to generate.
+ :param in_data: Data to process.
+ :type table: pandas.Series
+ :type in_data: InputData
+ """
+ logging.info(f" Generating the table {table.get(u'title', u'')} ...")
+
+ # Transform the data
+ logging.info(
+ f" Creating the data set for the {table.get(u'type', u'')} "
+ f"{table.get(u'title', u'')}."
+ )
+
+ incl_tests = table.get(u"include-tests", None)
+ if incl_tests not in (u"NDR", u"PDR"):
+ logging.error(f"Wrong tests to include specified ({incl_tests}).")
+ return
+
+ nr_cols = table.get(u"nr-of-data-columns", None)
+ if not nr_cols or nr_cols < 2:
+ logging.error(
+ f"No columns specified for {table.get(u'title', u'')}. Skipping."
+ )
+ return
+
+ data = in_data.filter_data(
+ table,
+ params=[u"throughput", u"result", u"name", u"parent", u"tags"],
+ continue_on_error=True
+ )
+
+ header = [
+ [u"VPP Version", ],
+ [u"Start Timestamp", ],
+ [u"CSIT Build", ],
+ [u"CSIT Testbed", ]
+ ]
+ tbl_dict = dict()
+ idx = 0
+ tb_tbl = table.get(u"testbeds", None)
+ for job_name, job_data in data.items():
+ for build_nr, build in job_data.items():
+ if idx >= nr_cols:
+ break
+ if build.empty:
+ continue
+
+ tb_ip = in_data.metadata(job_name, build_nr).get(u"testbed", u"")
+ if tb_ip and tb_tbl:
+ testbed = tb_tbl.get(tb_ip, u"")
+ else:
+ testbed = u""
+ header[2].insert(1, build_nr)
+ header[3].insert(1, testbed)
+ header[1].insert(
+ 1, in_data.metadata(job_name, build_nr).get(u"generated", u"")
+ )
+ header[0].insert(
+ 1, in_data.metadata(job_name, build_nr).get(u"version", u"")
+ )
+
+ for tst_name, tst_data in build.items():
+ tst_name_mod = \
+ _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
+ if not tbl_dict.get(tst_name_mod, None):
+ tbl_dict[tst_name_mod] = dict(
+ name=tst_data[u'name'].rsplit(u'-', 1)[0],
+ )
+ try:
+ tbl_dict[tst_name_mod][-idx - 1] = \
+ tst_data[u"throughput"][incl_tests][u"LOWER"]
+ except (TypeError, IndexError, KeyError, ValueError):
+ pass
+ idx += 1
+
+ if idx < nr_cols:
+ logging.error(u"Not enough data to build the table! Skipping")
+ return
+
+ cmp_dict = dict()
+ for idx, cmp in enumerate(table.get(u"comparisons", list())):
+ idx_ref = cmp.get(u"reference", None)
+ idx_cmp = cmp.get(u"compare", None)
+ if idx_ref is None or idx_cmp is None:
+ continue
+ header[0].append(
+ f"Diff({header[0][idx_ref - idx].split(u'~')[-1]} vs "
+ f"{header[0][idx_cmp - idx].split(u'~')[-1]})"
+ )
+ header[1].append(u"")
+ header[2].append(u"")
+ header[3].append(u"")
+ for tst_name, tst_data in tbl_dict.items():
+ if not cmp_dict.get(tst_name, None):
+ cmp_dict[tst_name] = list()
+ ref_data = tst_data.get(idx_ref, None)
+ cmp_data = tst_data.get(idx_cmp, None)
+ if ref_data is None or cmp_data is None:
+ cmp_dict[tst_name].append(float(u'nan'))
+ else:
+ cmp_dict[tst_name].append(
+ relative_change(ref_data, cmp_data)
+ )
+
+ tbl_lst_none = list()
+ tbl_lst = list()
+ for tst_name, tst_data in tbl_dict.items():
+ itm_lst = [tst_data[u"name"], ]
+ for idx in range(nr_cols):
+ item = tst_data.get(-idx - 1, None)
+ if item is None:
+ itm_lst.insert(1, None)
+ else:
+ itm_lst.insert(1, round(item / 1e6, 1))
+ itm_lst.extend(
+ [
+ None if itm is None else round(itm, 1)
+ for itm in cmp_dict[tst_name]
+ ]
+ )
+ if str(itm_lst[-1]) == u"nan" or itm_lst[-1] is None:
+ tbl_lst_none.append(itm_lst)
+ else:
+ tbl_lst.append(itm_lst)
+
+ tbl_lst_none.sort(key=lambda rel: rel[0], reverse=False)
+ tbl_lst.sort(key=lambda rel: rel[0], reverse=False)
+ tbl_lst.sort(key=lambda rel: rel[-1], reverse=False)
+ tbl_lst.extend(tbl_lst_none)
+
+ # Generate csv table:
+ csv_file_name = f"{table[u'output-file']}.csv"
+ logging.info(f" Writing the file {csv_file_name}")
+ with open(csv_file_name, u"wt", encoding='utf-8') as file_handler:
+ for hdr in header:
+ file_handler.write(u",".join(hdr) + u"\n")
+ for test in tbl_lst:
+ file_handler.write(u",".join(
+ [
+ str(item).replace(u"None", u"-").replace(u"nan", u"-").
+ replace(u"null", u"-") for item in test
+ ]
+ ) + u"\n")
+
+ txt_file_name = f"{table[u'output-file']}.txt"
+ logging.info(f" Writing the file {txt_file_name}")
+ convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u",")
+
+ # Reorganize header in txt table
+ txt_table = list()
+ with open(txt_file_name, u"rt", encoding='utf-8') as file_handler:
+ for line in list(file_handler):
+ txt_table.append(line)
+ try:
+ txt_table.insert(5, txt_table.pop(2))
+ with open(txt_file_name, u"wt", encoding='utf-8') as file_handler:
+ file_handler.writelines(txt_table)
+ except IndexError:
+ pass
+
+ # Generate html table:
+ hdr_html = [
+ u"
".join(row) for row in zip(*header)
+ ]
+ _tpc_generate_html_table(
+ hdr_html,
+ tbl_lst,
+ table[u'output-file'],
+ sort_data=True,
+ title=table.get(u"title", u""),
+ generate_rst=False
+ )