X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=2a366b94db6e966a0fd548ec6d6e40f621f37dd8;hp=b931344b4b03a359c6929f8d3ff584475e5f6dc9;hb=e623a04c190479ebfd0f5af69b9c129d22456e3c;hpb=cbfa26dc0f5334bcd367c161b4eaad342355bbde

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index b931344b4b..2a366b94db 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -34,7 +34,7 @@ from pal_utils import mean, stdev, relative_change, classify_anomalies, \
     convert_csv_to_pretty_txt, relative_change_stdev
 
 
-REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*')
+REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
 
 
 def generate_tables(spec, data):
@@ -47,7 +47,6 @@ def generate_tables(spec, data):
     """
 
     generator = {
-        u"table_details": table_details,
         u"table_merged_details": table_merged_details,
         u"table_perf_comparison": table_perf_comparison,
         u"table_perf_comparison_nic": table_perf_comparison_nic,
@@ -57,7 +56,8 @@ def generate_tables(spec, data):
         u"table_perf_trending_dash_html": table_perf_trending_dash_html,
         u"table_last_failed_tests": table_last_failed_tests,
         u"table_failed_tests": table_failed_tests,
-        u"table_failed_tests_html": table_failed_tests_html
+        u"table_failed_tests_html": table_failed_tests_html,
+        u"table_oper_data_html": table_oper_data_html
     }
 
     logging.info(u"Generating the tables ...")
@@ -72,8 +72,8 @@ def generate_tables(spec, data):
     logging.info(u"Done.")
 
 
-def table_details(table, input_data):
-    """Generate the table(s) with algorithm: table_detailed_test_results
+def table_oper_data_html(table, input_data):
+    """Generate the table(s) with algorithm: html_table_oper_data
     specified in the specification file.
 
     :param table: Table to generate.
@@ -83,66 +83,196 @@ def table_details(table, input_data):
     """
 
     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
-
     # Transform the data
     logging.info(
         f"    Creating the data set for the {table.get(u'type', u'')} "
         f"{table.get(u'title', u'')}."
     )
-    data = input_data.filter_data(table)
+    data = input_data.filter_data(
+        table,
+        params=[u"name", u"parent", u"show-run", u"type"],
+        continue_on_error=True
+    )
+    if data.empty:
+        return
+    data = input_data.merge_data(data)
 
-    # Prepare the header of the tables
-    header = list()
-    for column in table[u"columns"]:
-        header.append(
-            u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
+    sort_tests = table.get(u"sort", None)
+    if sort_tests:
+        args = dict(
+            inplace=True,
+            ascending=(sort_tests == u"ascending")
         )
+        data.sort_index(**args)
 
-    # Generate the data for the table according to the model in the table
-    # specification
-    job = list(table[u"data"].keys())[0]
-    build = str(table[u"data"][job][0])
-    try:
-        suites = input_data.suites(job, build)
-    except KeyError:
-        logging.error(
-            u"    No data available. The table will not be generated."
-        )
+    suites = input_data.filter_data(
+        table,
+        continue_on_error=True,
+        data_set=u"suites"
+    )
+    if suites.empty:
         return
+    suites = input_data.merge_data(suites)
 
-    for suite in suites.values:
-        # Generate data
-        suite_name = suite[u"name"]
-        table_lst = list()
-        for test in data[job][build].keys():
-            if data[job][build][test][u"parent"] not in suite_name:
+    def _generate_html_table(tst_data):
+        """Generate an HTML table with operational data for the given test.
+
+        :param tst_data: Test data to be used to generate the table.
+        :type tst_data: pandas.Series
+        :returns: HTML table with operational data.
+        :rtype: str
+        """
+
+        colors = {
+            u"header": u"#7eade7",
+            u"empty": u"#ffffff",
+            u"body": (u"#e9f1fb", u"#d4e4f7")
+        }
+
+        tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
+
+        trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
+        thead = ET.SubElement(
+            trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+        )
+        thead.text = tst_data[u"name"]
+
+        trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
+        thead = ET.SubElement(
+            trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+        )
+        thead.text = u"\t"
+
+        if tst_data.get(u"show-run", u"No Data") == u"No Data":
+            trow = ET.SubElement(
+                tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
+            )
+            tcol = ET.SubElement(
+                trow, u"td", attrib=dict(align=u"left", colspan=u"6")
+            )
+            tcol.text = u"No Data"
+
+            trow = ET.SubElement(
+                tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
+            )
+            thead = ET.SubElement(
+                trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+            )
+            font = ET.SubElement(
+                thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
+            )
+            font.text = u"."
+            return str(ET.tostring(tbl, encoding=u"unicode"))
+
+        tbl_hdr = (
+            u"Name",
+            u"Nr of Vectors",
+            u"Nr of Packets",
+            u"Suspends",
+            u"Cycles per Packet",
+            u"Average Vector Size"
+        )
+
+        for dut_data in tst_data[u"show-run"].values():
+            trow = ET.SubElement(
+                tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
+            )
+            tcol = ET.SubElement(
+                trow, u"td", attrib=dict(align=u"left", colspan=u"6")
+            )
+            if dut_data.get(u"threads", None) is None:
+                tcol.text = u"No Data"
                 continue
-            row_lst = list()
-            for column in table[u"columns"]:
-                try:
-                    col_data = str(data[job][build][test][column[
-                        u"data"].split(" ")[1]]).replace(u'"', u'""')
-                    if column[u"data"].split(u" ")[1] in \
-                        (u"conf-history", u"show-run"):
-                        col_data = col_data.replace(u" |br| ", u"", )
-                        col_data = f" |prein| {col_data[:-5]} |preout| "
-                    row_lst.append(f'"{col_data}"')
-                except KeyError:
-                    row_lst.append(u"No data")
-            table_lst.append(row_lst)
 
-        # Write the data to file
-        if table_lst:
-            file_name = (
-                f"{table[u'output-file']}_{suite_name}"
-                f"{table[u'output-file-ext']}"
+            bold = ET.SubElement(tcol, u"b")
+            bold.text = (
+                f"Host IP: {dut_data.get(u'host', '')}, "
+                f"Socket: {dut_data.get(u'socket', '')}"
             )
-            logging.info(f"      Writing file: {file_name}")
-            with open(file_name, u"w") as file_handler:
-                file_handler.write(u",".join(header) + u"\n")
-                for item in table_lst:
-                    file_handler.write(u",".join(item) + u"\n")
+            trow = ET.SubElement(
+                tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
+            )
+            thead = ET.SubElement(
+                trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+            )
+            thead.text = u"\t"
 
+            for thread_nr, thread in dut_data[u"threads"].items():
+                trow = ET.SubElement(
+                    tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
+                )
+                tcol = ET.SubElement(
+                    trow, u"td", attrib=dict(align=u"left", colspan=u"6")
+                )
+                bold = ET.SubElement(tcol, u"b")
+                bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
+                trow = ET.SubElement(
+                    tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
+                )
+                for idx, col in enumerate(tbl_hdr):
+                    tcol = ET.SubElement(
+                        trow, u"td",
+                        attrib=dict(align=u"right" if idx else u"left")
+                    )
+                    font = ET.SubElement(
+                        tcol, u"font", attrib=dict(size=u"2")
+                    )
+                    bold = ET.SubElement(font, u"b")
+                    bold.text = col
+                for row_nr, row in enumerate(thread):
+                    trow = ET.SubElement(
+                        tbl, u"tr",
+                        attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
+                    )
+                    for idx, col in enumerate(row):
+                        tcol = ET.SubElement(
+                            trow, u"td",
+                            attrib=dict(align=u"right" if idx else u"left")
+                        )
+                        font = ET.SubElement(
+                            tcol, u"font", attrib=dict(size=u"2")
+                        )
+                        if isinstance(col, float):
+                            font.text = f"{col:.2f}"
+                        else:
+                            font.text = str(col)
+                trow = ET.SubElement(
+                    tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
+                )
+                thead = ET.SubElement(
+                    trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+                )
+                thead.text = u"\t"
+
+        trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
+        thead = ET.SubElement(
+            trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+        )
+        font = ET.SubElement(
+            thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
+        )
+        font.text = u"."
+
+        return str(ET.tostring(tbl, encoding=u"unicode"))
+
+    for suite in suites.values:
+        html_table = str()
+        for test_data in data.values:
+            if test_data[u"parent"] not in suite[u"name"]:
+                continue
+            html_table += _generate_html_table(test_data)
+        if not html_table:
+            continue
+        try:
+            file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
+            with open(f"{file_name}", u'w') as html_file:
+                logging.info(f"    Writing file: {file_name}")
+                html_file.write(u".. raw:: html\n\n\t")
+                html_file.write(html_table)
+                html_file.write(u"\n\t<p><br><br></p>\n")
+        except KeyError:
+            logging.warning(u"The output file is not defined.")
+            return
     logging.info(u"  Done.")
 
 
@@ -157,6 +287,7 @@ def table_merged_details(table, input_data):
     """
 
     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
+
     # Transform the data
     logging.info(
         f"    Creating the data set for the {table.get(u'type', u'')} "
@@ -164,12 +295,15 @@ def table_merged_details(table, input_data):
     )
     data = input_data.filter_data(table, continue_on_error=True)
     data = input_data.merge_data(data)
-    data.sort_index(inplace=True)
 
-    logging.info(
-        f"    Creating the data set for the {table.get(u'type', u'')} "
-        f"{table.get(u'title', u'')}."
-    )
+    sort_tests = table.get(u"sort", None)
+    if sort_tests:
+        args = dict(
+            inplace=True,
+            ascending=(sort_tests == u"ascending")
+        )
+        data.sort_index(**args)
+
     suites = input_data.filter_data(
         table, continue_on_error=True, data_set=u"suites")
     suites = input_data.merge_data(suites)
@@ -193,27 +327,45 @@ def table_merged_details(table, input_data):
                 try:
                     col_data = str(data[test][column[
                         u"data"].split(u" ")[1]]).replace(u'"', u'""')
+                    # Do not include tests with "Test Failed" in test message
+                    if u"Test Failed" in col_data:
+                        continue
                     col_data = col_data.replace(
                         u"No Data", u"Not Captured     "
                     )
-                    if column[u"data"].split(u" ")[1] in \
-                        (u"conf-history", u"show-run"):
+                    if column[u"data"].split(u" ")[1] in (u"name", ):
+                        if len(col_data) > 30:
+                            col_data_lst = col_data.split(u"-")
+                            half = int(len(col_data_lst) / 2)
+                            col_data = f"{u'-'.join(col_data_lst[:half])}" \
+                                       f"- |br| " \
+                                       f"{u'-'.join(col_data_lst[half:])}"
+                        col_data = f" |prein| {col_data} |preout| "
+                    elif column[u"data"].split(u" ")[1] in (u"msg", ):
+                        # Temporary solution: remove NDR results from message:
+                        if bool(table.get(u'remove-ndr', False)):
+                            try:
+                                col_data = col_data.split(u" |br| ", 1)[1]
+                            except IndexError:
+                                pass
+                        col_data = f" |prein| {col_data} |preout| "
+                    elif column[u"data"].split(u" ")[1] in \
+                            (u"conf-history", u"show-run"):
                         col_data = col_data.replace(u" |br| ", u"", 1)
                         col_data = f" |prein| {col_data[:-5]} |preout| "
                     row_lst.append(f'"{col_data}"')
                 except KeyError:
                     row_lst.append(u'"Not captured"')
-            table_lst.append(row_lst)
+            if len(row_lst) == len(table[u"columns"]):
+                table_lst.append(row_lst)
 
         # Write the data to file
         if table_lst:
-            file_name = (
-                f"{table[u'output-file']}_{suite_name}"
-                f"{table[u'output-file-ext']}"
-            )
+            separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
+            file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
             logging.info(f"      Writing file: {file_name}")
-            with open(file_name, u"w") as file_handler:
-                file_handler.write(u",".join(header) + "u\n")
+            with open(file_name, u"wt") as file_handler:
+                file_handler.write(u",".join(header) + u"\n")
                 for item in table_lst:
                     file_handler.write(u",".join(item) + u"\n")
 
@@ -469,13 +621,15 @@ def table_perf_comparison(table, input_data):
 
     # Prepare data to the table:
     tbl_dict = dict()
-    topo = ""
+    # topo = ""
     for job, builds in table[u"reference"][u"data"].items():
-        topo = u"2n-skx" if u"2n-skx" in job else u""
+        # topo = u"2n-skx" if u"2n-skx" in job else u""
         for build in builds:
             for tst_name, tst_data in data[job][str(build)].items():
                 tst_name_mod = _tpc_modify_test_name(tst_name)
-                if u"across topologies" in table[u"title"].lower():
+                if (u"across topologies" in table[u"title"].lower() or
+                        (u" 3n-" in table[u"title"].lower() and
+                         u" 2n-" in table[u"title"].lower())):
                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
                 if tbl_dict.get(tst_name_mod, None) is None:
                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
@@ -494,11 +648,47 @@ def table_perf_comparison(table, input_data):
                                  src=tst_data,
                                  include_tests=table[u"include-tests"])
 
+    replacement = table[u"reference"].get(u"data-replacement", None)
+    if replacement:
+        create_new_list = True
+        rpl_data = input_data.filter_data(
+            table, data=replacement, continue_on_error=True)
+        for job, builds in replacement.items():
+            for build in builds:
+                for tst_name, tst_data in rpl_data[job][str(build)].items():
+                    tst_name_mod = _tpc_modify_test_name(tst_name)
+                    if (u"across topologies" in table[u"title"].lower() or
+                            (u" 3n-" in table[u"title"].lower() and
+                             u" 2n-" in table[u"title"].lower())):
+                        tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
+                    if tbl_dict.get(tst_name_mod, None) is None:
+                        name = \
+                            f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
+                        if u"across testbeds" in table[u"title"].lower() or \
+                                u"across topologies" in table[u"title"].lower():
+                            name = _tpc_modify_displayed_test_name(name)
+                        tbl_dict[tst_name_mod] = {
+                            u"name": name,
+                            u"ref-data": list(),
+                            u"cmp-data": list()
+                        }
+                    if create_new_list:
+                        create_new_list = False
+                        tbl_dict[tst_name_mod][u"ref-data"] = list()
+
+                    _tpc_insert_data(
+                        target=tbl_dict[tst_name_mod][u"ref-data"],
+                        src=tst_data,
+                        include_tests=table[u"include-tests"]
+                    )
+
     for job, builds in table[u"compare"][u"data"].items():
         for build in builds:
             for tst_name, tst_data in data[job][str(build)].items():
                 tst_name_mod = _tpc_modify_test_name(tst_name)
-                if u"across topologies" in table[u"title"].lower():
+                if (u"across topologies" in table[u"title"].lower() or
+                        (u" 3n-" in table[u"title"].lower() and
+                         u" 2n-" in table[u"title"].lower())):
                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
                 if tbl_dict.get(tst_name_mod, None) is None:
                     groups = re.search(REGEX_NIC, tst_data[u"parent"])
@@ -528,7 +718,9 @@ def table_perf_comparison(table, input_data):
             for build in builds:
                 for tst_name, tst_data in rpl_data[job][str(build)].items():
                     tst_name_mod = _tpc_modify_test_name(tst_name)
-                    if u"across topologies" in table[u"title"].lower():
+                    if (u"across topologies" in table[u"title"].lower() or
+                            (u" 3n-" in table[u"title"].lower() and
+                             u" 2n-" in table[u"title"].lower())):
                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
                     if tbl_dict.get(tst_name_mod, None) is None:
                         name = \
@@ -556,7 +748,9 @@ def table_perf_comparison(table, input_data):
             for build in builds:
                 for tst_name, tst_data in data[job][str(build)].items():
                     tst_name_mod = _tpc_modify_test_name(tst_name)
-                    if u"across topologies" in table[u"title"].lower():
+                    if (u"across topologies" in table[u"title"].lower() or
+                            (u" 3n-" in table[u"title"].lower() and
+                             u" 2n-" in table[u"title"].lower())):
                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
                     if tbl_dict.get(tst_name_mod, None) is None:
                         continue
@@ -609,10 +803,10 @@ def table_perf_comparison(table, input_data):
         if item[-2] == u"Not tested":
             pass
         elif item[-4] == u"Not tested":
-            item.append(u"New in CSIT-1908")
-        elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
-            item.append(u"See footnote [1]")
-            footnote = True
+            item.append(u"New in CSIT-2001")
+        # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
+        #     item.append(u"See footnote [1]")
+        #     footnote = True
         elif item[-4] != 0:
             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
         if (len(item) == len(header)) and (item[-3] != u"Not tested"):
@@ -622,7 +816,7 @@ def table_perf_comparison(table, input_data):
 
     # Generate csv tables:
     csv_file = f"{table[u'output-file']}.csv"
-    with open(csv_file, u"w") as file_handler:
+    with open(csv_file, u"wt") as file_handler:
         file_handler.write(header_str)
         for test in tbl_lst:
             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
@@ -700,15 +894,17 @@ def table_perf_comparison_nic(table, input_data):
 
     # Prepare data to the table:
     tbl_dict = dict()
-    topo = u""
+    # topo = u""
     for job, builds in table[u"reference"][u"data"].items():
-        topo = u"2n-skx" if u"2n-skx" in job else u""
+        # topo = u"2n-skx" if u"2n-skx" in job else u""
         for build in builds:
             for tst_name, tst_data in data[job][str(build)].items():
                 if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
                     continue
                 tst_name_mod = _tpc_modify_test_name(tst_name)
-                if u"across topologies" in table[u"title"].lower():
+                if (u"across topologies" in table[u"title"].lower() or
+                        (u" 3n-" in table[u"title"].lower() and
+                         u" 2n-" in table[u"title"].lower())):
                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
                 if tbl_dict.get(tst_name_mod, None) is None:
                     name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
@@ -726,13 +922,51 @@ def table_perf_comparison_nic(table, input_data):
                     include_tests=table[u"include-tests"]
                 )
 
+    replacement = table[u"reference"].get(u"data-replacement", None)
+    if replacement:
+        create_new_list = True
+        rpl_data = input_data.filter_data(
+            table, data=replacement, continue_on_error=True)
+        for job, builds in replacement.items():
+            for build in builds:
+                for tst_name, tst_data in rpl_data[job][str(build)].items():
+                    if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
+                        continue
+                    tst_name_mod = _tpc_modify_test_name(tst_name)
+                    if (u"across topologies" in table[u"title"].lower() or
+                            (u" 3n-" in table[u"title"].lower() and
+                             u" 2n-" in table[u"title"].lower())):
+                        tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
+                    if tbl_dict.get(tst_name_mod, None) is None:
+                        name = \
+                            f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
+                        if u"across testbeds" in table[u"title"].lower() or \
+                                u"across topologies" in table[u"title"].lower():
+                            name = _tpc_modify_displayed_test_name(name)
+                        tbl_dict[tst_name_mod] = {
+                            u"name": name,
+                            u"ref-data": list(),
+                            u"cmp-data": list()
+                        }
+                    if create_new_list:
+                        create_new_list = False
+                        tbl_dict[tst_name_mod][u"ref-data"] = list()
+
+                    _tpc_insert_data(
+                        target=tbl_dict[tst_name_mod][u"ref-data"],
+                        src=tst_data,
+                        include_tests=table[u"include-tests"]
+                    )
+
     for job, builds in table[u"compare"][u"data"].items():
         for build in builds:
             for tst_name, tst_data in data[job][str(build)].items():
                 if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
                     continue
                 tst_name_mod = _tpc_modify_test_name(tst_name)
-                if u"across topologies" in table[u"title"].lower():
+                if (u"across topologies" in table[u"title"].lower() or
+                        (u" 3n-" in table[u"title"].lower() and
+                         u" 2n-" in table[u"title"].lower())):
                     tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
                 if tbl_dict.get(tst_name_mod, None) is None:
                     name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
@@ -761,7 +995,9 @@ def table_perf_comparison_nic(table, input_data):
                     if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
                         continue
                     tst_name_mod = _tpc_modify_test_name(tst_name)
-                    if u"across topologies" in table[u"title"].lower():
+                    if (u"across topologies" in table[u"title"].lower() or
+                            (u" 3n-" in table[u"title"].lower() and
+                             u" 2n-" in table[u"title"].lower())):
                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
                     if tbl_dict.get(tst_name_mod, None) is None:
                         name = \
@@ -791,7 +1027,9 @@ def table_perf_comparison_nic(table, input_data):
                     if item[u"nic"] not in tst_data[u"tags"]:
                         continue
                     tst_name_mod = _tpc_modify_test_name(tst_name)
-                    if u"across topologies" in table[u"title"].lower():
+                    if (u"across topologies" in table[u"title"].lower() or
+                            (u" 3n-" in table[u"title"].lower() and
+                             u" 2n-" in table[u"title"].lower())):
                         tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
                     if tbl_dict.get(tst_name_mod, None) is None:
                         continue
@@ -844,10 +1082,10 @@ def table_perf_comparison_nic(table, input_data):
         if item[-2] == u"Not tested":
             pass
         elif item[-4] == u"Not tested":
-            item.append(u"New in CSIT-1908")
-        elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
-            item.append(u"See footnote [1]")
-            footnote = True
+            item.append(u"New in CSIT-2001")
+        # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
+        #     item.append(u"See footnote [1]")
+        #     footnote = True
         elif item[-4] != 0:
             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
         if (len(item) == len(header)) and (item[-3] != u"Not tested"):
@@ -857,7 +1095,7 @@ def table_perf_comparison_nic(table, input_data):
 
     # Generate csv tables:
     csv_file = f"{table[u'output-file']}.csv"
-    with open(csv_file, u"w") as file_handler:
+    with open(csv_file, u"wt") as file_handler:
         file_handler.write(header_str)
         for test in tbl_lst:
             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
@@ -983,7 +1221,7 @@ def table_nics_comparison(table, input_data):
     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 
     # Generate csv tables:
-    with open(f"{table[u'output-file']}.csv", u"w") as file_handler:
+    with open(f"{table[u'output-file']}.csv", u"wt") as file_handler:
         file_handler.write(u",".join(header) + u"\n")
         for test in tbl_lst:
             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
@@ -1117,7 +1355,7 @@ def table_soak_vs_ndr(table, input_data):
 
     # Generate csv tables:
     csv_file = f"{table[u'output-file']}.csv"
-    with open(csv_file, u"w") as file_handler:
+    with open(csv_file, u"wt") as file_handler:
         file_handler.write(header_str)
         for test in tbl_lst:
             file_handler.write(u",".join([str(item) for item in test]) + u"\n")
@@ -1240,7 +1478,7 @@ def table_perf_trending_dash(table, input_data):
     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
 
     logging.info(f"    Writing file: {file_name}")
-    with open(file_name, u"w") as file_handler:
+    with open(file_name, u"wt") as file_handler:
         file_handler.write(header_str)
         for test in tbl_sorted:
             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
@@ -1249,13 +1487,11 @@ def table_perf_trending_dash(table, input_data):
     convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
 
 
-def _generate_url(base, testbed, test_name):
+def _generate_url(testbed, test_name):
     """Generate URL to a trending plot from the name of the test case.
 
-    :param base: The base part of URL common to all test cases.
     :param testbed: The testbed used for testing.
     :param test_name: The name of the test case.
-    :type base: str
     :type testbed: str
     :type test_name: str
     :returns: The URL to the plot with the trending data for the given test
@@ -1263,140 +1499,140 @@ def _generate_url(base, testbed, test_name):
     :rtype str
     """
 
-    url = base
-    file_name = u""
-    anchor = u".html#"
-    feature = u""
-
-    if u"lbdpdk" in test_name or u"lbvpp" in test_name:
-        file_name = u"link_bonding"
-
-    elif u"114b" in test_name and u"vhost" in test_name:
-        file_name = u"vts"
-
-    elif u"testpmd" in test_name or u"l3fwd" in test_name:
-        file_name = u"dpdk"
-
-    elif u"memif" in test_name:
-        file_name = u"container_memif"
-        feature = u"-base"
-
-    elif u"srv6" in test_name:
-        file_name = u"srv6"
-
-    elif u"vhost" in test_name:
-        if u"l2xcbase" in test_name or u"l2bdbasemaclrn" in test_name:
-            file_name = u"vm_vhost_l2"
-            if u"114b" in test_name:
-                feature = u""
-            elif u"l2xcbase" in test_name and u"x520" in test_name:
-                feature = u"-base-l2xc"
-            elif u"l2bdbasemaclrn" in test_name and u"x520" in test_name:
-                feature = u"-base-l2bd"
-            else:
-                feature = u"-base"
-        elif u"ip4base" in test_name:
-            file_name = u"vm_vhost_ip4"
-            feature = u"-base"
-
-    elif u"ipsecbasetnlsw" in test_name:
-        file_name = u"ipsecsw"
-        feature = u"-base-scale"
-
-    elif u"ipsec" in test_name:
-        file_name = u"ipsec"
-        feature = u"-base-scale"
-        if u"hw-" in test_name:
-            file_name = u"ipsechw"
-        elif u"sw-" in test_name:
-            file_name = u"ipsecsw"
-        if u"-int-" in test_name:
-            feature = u"-base-scale-int"
-        elif u"tnl" in test_name:
-            feature = u"-base-scale-tnl"
-
-    elif u"ethip4lispip" in test_name or u"ethip4vxlan" in test_name:
-        file_name = u"ip4_tunnels"
-        feature = u"-base"
-
-    elif u"ip4base" in test_name or u"ip4scale" in test_name:
-        file_name = u"ip4"
-        if u"xl710" in test_name:
-            feature = u"-base-scale-features"
-        elif u"iacl" in test_name:
-            feature = u"-features-iacl"
-        elif u"oacl" in test_name:
-            feature = u"-features-oacl"
-        elif u"snat" in test_name or u"cop" in test_name:
-            feature = u"-features"
-        else:
-            feature = u"-base-scale"
-
-    elif u"ip6base" in test_name or u"ip6scale" in test_name:
-        file_name = u"ip6"
-        feature = u"-base-scale"
-
-    elif u"l2xcbase" in test_name or u"l2xcscale" in test_name \
-            or u"l2bdbasemaclrn" in test_name or u"l2bdscale" in test_name:
-        file_name = u"l2"
-        if u"macip" in test_name:
-            feature = u"-features-macip"
-        elif u"iacl" in test_name:
-            feature = u"-features-iacl"
-        elif u"oacl" in test_name:
-            feature = u"-features-oacl"
-        else:
-            feature = u"-base-scale"
-
     if u"x520" in test_name:
-        nic = u"x520-"
+        nic = u"x520"
     elif u"x710" in test_name:
-        nic = u"x710-"
+        nic = u"x710"
     elif u"xl710" in test_name:
-        nic = u"xl710-"
+        nic = u"xl710"
     elif u"xxv710" in test_name:
-        nic = u"xxv710-"
+        nic = u"xxv710"
     elif u"vic1227" in test_name:
-        nic = u"vic1227-"
+        nic = u"vic1227"
     elif u"vic1385" in test_name:
-        nic = u"vic1385-"
+        nic = u"vic1385"
     elif u"x553" in test_name:
-        nic = u"x553-"
+        nic = u"x553"
+    elif u"cx556" in test_name or u"cx556a" in test_name:
+        nic = u"cx556a"
     else:
         nic = u""
-    anchor += nic
 
     if u"64b" in test_name:
-        framesize = u"64b"
+        frame_size = u"64b"
     elif u"78b" in test_name:
-        framesize = u"78b"
+        frame_size = u"78b"
     elif u"imix" in test_name:
-        framesize = u"imix"
+        frame_size = u"imix"
     elif u"9000b" in test_name:
-        framesize = u"9000b"
+        frame_size = u"9000b"
     elif u"1518b" in test_name:
-        framesize = u"1518b"
+        frame_size = u"1518b"
     elif u"114b" in test_name:
-        framesize = u"114b"
+        frame_size = u"114b"
+    else:
+        frame_size = u""
+
+    if u"1t1c" in test_name or \
+        (u"-1c-" in test_name and
+         testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
+        cores = u"1t1c"
+    elif u"2t2c" in test_name or \
+         (u"-2c-" in test_name and
+          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
+        cores = u"2t2c"
+    elif u"4t4c" in test_name or \
+         (u"-4c-" in test_name and
+          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
+        cores = u"4t4c"
+    elif u"2t1c" in test_name or \
+         (u"-1c-" in test_name and
+          testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
+        cores = u"2t1c"
+    elif u"4t2c" in test_name or \
+         (u"-2c-" in test_name and
+          testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
+        cores = u"4t2c"
+    elif u"8t4c" in test_name or \
+         (u"-4c-" in test_name and
+          testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
+        cores = u"8t4c"
+    else:
+        cores = u""
+
+    if u"testpmd" in test_name:
+        driver = u"testpmd"
+    elif u"l3fwd" in test_name:
+        driver = u"l3fwd"
+    elif u"avf" in test_name:
+        driver = u"avf"
+    elif u"rdma" in test_name:
+        driver = u"rdma"
+    elif u"dnv" in testbed or u"tsh" in testbed:
+        driver = u"ixgbe"
+    else:
+        driver = u"dpdk"
+
+    if u"acl" in test_name or \
+            u"macip" in test_name or \
+            u"nat" in test_name or \
+            u"policer" in test_name or \
+            u"cop" in test_name:
+        bsf = u"features"
+    elif u"scale" in test_name:
+        bsf = u"scale"
+    elif u"base" in test_name:
+        bsf = u"base"
+    else:
+        bsf = u"base"
+
+    if u"114b" in test_name and u"vhost" in test_name:
+        domain = u"vts"
+    elif u"testpmd" in test_name or u"l3fwd" in test_name:
+        domain = u"dpdk"
+    elif u"memif" in test_name:
+        domain = u"container_memif"
+    elif u"srv6" in test_name:
+        domain = u"srv6"
+    elif u"vhost" in test_name:
+        domain = u"vhost"
+        if u"vppl2xc" in test_name:
+            driver += u"-vpp"
+        else:
+            driver += u"-testpmd"
+        if u"lbvpplacp" in test_name:
+            bsf += u"-link-bonding"
+    elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
+        domain = u"nf_service_density_vnfc"
+    elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
+        domain = u"nf_service_density_cnfc"
+    elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
+        domain = u"nf_service_density_cnfp"
+    elif u"ipsec" in test_name:
+        domain = u"ipsec"
+        if u"sw" in test_name:
+            bsf += u"-sw"
+        elif u"hw" in test_name:
+            bsf += u"-hw"
+    elif u"ethip4vxlan" in test_name:
+        domain = u"ip4_tunnels"
+    elif u"ip4base" in test_name or u"ip4scale" in test_name:
+        domain = u"ip4"
+    elif u"ip6base" in test_name or u"ip6scale" in test_name:
+        domain = u"ip6"
+    elif u"l2xcbase" in test_name or \
+            u"l2xcscale" in test_name or \
+            u"l2bdbasemaclrn" in test_name or \
+            u"l2bdscale" in test_name or \
+            u"l2patch" in test_name:
+        domain = u"l2"
     else:
-        framesize = u""
-    anchor += framesize + u"-"
-
-    if u"1t1c" in test_name:
-        anchor += u"1t1c"
-    elif u"2t2c" in test_name:
-        anchor += u"2t2c"
-    elif u"4t4c" in test_name:
-        anchor += u"4t4c"
-    elif u"2t1c" in test_name:
-        anchor += u"2t1c"
-    elif u"4t2c" in test_name:
-        anchor += u"4t2c"
-    elif u"8t4c" in test_name:
-        anchor += u"8t4c"
-
-    return url + file_name + u"-" + testbed + u"-" + nic + framesize + \
-        feature.replace("-int", u"").replace("-tnl", u"") + anchor + feature
+        domain = u""
+
+    file_name = u"-".join((domain, testbed, nic)) + u".html#"
+    anchor_name = u"-".join((frame_size, cores, bsf, driver))
+
+    return file_name + anchor_name
 
 
 def table_perf_trending_dash_html(table, input_data):
@@ -1483,11 +1719,8 @@ def table_perf_trending_dash_html(table, input_data):
                     tdata,
                     u"a",
                     attrib=dict(
-                        href=_generate_url(
-                            u"../trending/",
-                            table.get(u"testbed", None),
-                            item
-                        )
+                        href=f"../trending/"
+                             f"{_generate_url(table.get(u'testbed', ''), item)}"
                     )
                 )
                 ref.text = item
@@ -1561,7 +1794,7 @@ def table_last_failed_tests(table, input_data):
 
     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
     logging.info(f"    Writing file: {file_name}")
-    with open(file_name, u"w") as file_handler:
+    with open(file_name, u"wt") as file_handler:
         for test in tbl_list:
             file_handler.write(test + u'\n')
 
@@ -1666,7 +1899,7 @@ def table_failed_tests(table, input_data):
 
     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
     logging.info(f"    Writing file: {file_name}")
-    with open(file_name, u"w") as file_handler:
+    with open(file_name, u"wt") as file_handler:
         file_handler.write(u",".join(header) + u"\n")
         for test in tbl_sorted:
             file_handler.write(u",".join([str(item) for item in test]) + u'\n')
@@ -1740,11 +1973,8 @@ def table_failed_tests_html(table, input_data):
                     tdata,
                     u"a",
                     attrib=dict(
-                        href=_generate_url(
-                            u"../trending/",
-                            table.get(u"testbed", None),
-                            item
-                        )
+                        href=f"../trending/"
+                             f"{_generate_url(table.get(u'testbed', ''), item)}"
                     )
                 )
                 ref.text = item