Report: Add RCA to comp tables

For now, only these comaprison tables:
- table_perf_comparison_nic
- table_perf_comparison

Change-Id: I52c5a960eafbe2bbf11f6c371d4a72419373dc76
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 5cd3553..1366ea0 100644 (file)
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -29,6 +29,7 @@ import plotly.offline as ploff
 import pandas as pd
 
 from numpy import nan, isnan
+from yaml import load, FullLoader, YAMLError
 
 from pal_utils import mean, stdev, classify_anomalies, \
     convert_csv_to_pretty_txt, relative_change_stdev
@@ -492,14 +493,24 @@ def _tpc_generate_html_table(header, data, output_file_name):
     :type output_file_name: str
     """
 
+    try:
+        idx = header.index(u"Test case")
+    except ValueError:
+        idx = 0
+    params = {
+        u"align-hdr": ([u"left", u"center"], [u"left", u"left", u"center"]),
+        u"align-itm": ([u"left", u"right"], [u"left", u"left", u"right"]),
+        u"width": ([28, 9], [4, 24, 10])
+    }
+
     df_data = pd.DataFrame(data, columns=header)
 
     df_sorted = [df_data.sort_values(
-        by=[key, header[0]], ascending=[True, True]
-        if key != header[0] else [False, True]) for key in header]
+        by=[key, header[idx]], ascending=[True, True]
+        if key != header[idx] else [False, True]) for key in header]
     df_sorted_rev = [df_data.sort_values(
-        by=[key, header[0]], ascending=[False, True]
-        if key != header[0] else [True, True]) for key in header]
+        by=[key, header[idx]], ascending=[False, True]
+        if key != header[idx] else [True, True]) for key in header]
     df_sorted.extend(df_sorted_rev)
 
     fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
@@ -507,7 +518,7 @@ def _tpc_generate_html_table(header, data, output_file_name):
     table_header = dict(
         values=[f"<b>{item}</b>" for item in header],
         fill_color=u"#7eade7",
-        align=[u"left", u"center"]
+        align=params[u"align-hdr"][idx]
     )
 
     fig = go.Figure()
@@ -516,12 +527,12 @@ def _tpc_generate_html_table(header, data, output_file_name):
         columns = [table.get(col) for col in header]
         fig.add_trace(
             go.Table(
-                columnwidth=[30, 10],
+                columnwidth=params[u"width"][idx],
                 header=table_header,
                 cells=dict(
                     values=columns,
                     fill_color=fill_color,
-                    align=[u"left", u"right"]
+                    align=params[u"align-itm"][idx]
                 )
             )
         )
@@ -550,7 +561,7 @@ def _tpc_generate_html_table(header, data, output_file_name):
                 xanchor=u"left",
                 y=1.045,
                 yanchor=u"top",
-                active=len(menu_items) - 2,
+                active=len(menu_items) - 1,
                 buttons=list(buttons)
             )
         ],
@@ -593,6 +604,16 @@ def table_perf_comparison(table, input_data):
     try:
         header = [u"Test case", ]
 
+        rca_data = None
+        rca = table.get(u"rca", None)
+        if rca:
+            try:
+                with open(rca.get(u"data-file", ""), u"r") as rca_file:
+                    rca_data = load(rca_file, Loader=FullLoader)
+                header.insert(0, rca.get(u"title", "RCA"))
+            except (YAMLError, IOError) as err:
+                logging.warning(repr(err))
+
         if table[u"include-tests"] == u"MRR":
             hdr_param = u"Rec Rate"
         else:
@@ -616,16 +637,14 @@ def table_perf_comparison(table, input_data):
                 u"Stdev of delta [%]"
             ]
         )
-        header_str = u",".join(header) + u"\n"
+        header_str = u";".join(header) + u"\n"
     except (AttributeError, KeyError) as err:
         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
         return
 
     # Prepare data to the table:
     tbl_dict = dict()
-    # topo = ""
     for job, builds in table[u"reference"][u"data"].items():
-        # topo = u"2n-skx" if u"2n-skx" in job else u""
         for build in builds:
             for tst_name, tst_data in data[job][str(build)].items():
                 tst_name_mod = _tpc_modify_test_name(tst_name)
@@ -777,7 +796,6 @@ def table_perf_comparison(table, input_data):
                         pass
 
     tbl_lst = list()
-    footnote = False
     for tst_name in tbl_dict:
         item = [tbl_dict[tst_name][u"name"], ]
         if history:
@@ -815,9 +833,6 @@ def table_perf_comparison(table, input_data):
         elif item[-4] == u"Not tested":
             item.append(u"New in CSIT-2001")
             item.append(u"New in CSIT-2001")
-        # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
-        #     item.append(u"See footnote [1]")
-        #     footnote = True
         elif data_r_mean and data_c_mean:
             delta, d_stdev = relative_change_stdev(
                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
@@ -830,6 +845,8 @@ def table_perf_comparison(table, input_data):
                 item.append(round(d_stdev))
             except ValueError:
                 item.append(d_stdev)
+        if rca_data:
+            item.insert(0, rca_data.get(item[0], u" "))
         if (len(item) == len(header)) and (item[-4] != u"Not tested"):
             tbl_lst.append(item)
 
@@ -840,24 +857,16 @@ def table_perf_comparison(table, input_data):
     with open(csv_file, u"wt") as file_handler:
         file_handler.write(header_str)
         for test in tbl_lst:
-            file_handler.write(u",".join([str(item) for item in test]) + u"\n")
+            file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 
     txt_file_name = f"{table[u'output-file']}.txt"
     convert_csv_to_pretty_txt(csv_file, txt_file_name)
 
-    if footnote:
-        with open(txt_file_name, u'a') as txt_file:
-            txt_file.writelines([
-                u"\nFootnotes:\n",
-                u"[1] CSIT-1908 changed test methodology of dot1q tests in "
-                u"2-node testbeds, dot1q encapsulation is now used on both "
-                u"links of SUT.\n",
-                u"    Previously dot1q was used only on a single link with the "
-                u"other link carrying untagged Ethernet frames. This changes "
-                u"results\n",
-                u"    in slightly lower throughput in CSIT-1908 for these "
-                u"tests. See release notes."
-            ])
+    if rca_data:
+        footnote = rca_data.get(u"footnote", "")
+        if footnote:
+            with open(txt_file_name, u'a') as txt_file:
+                txt_file.writelines(footnote)
 
     # Generate html table:
     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
@@ -886,6 +895,16 @@ def table_perf_comparison_nic(table, input_data):
     try:
         header = [u"Test case", ]
 
+        rca_data = None
+        rca = table.get(u"rca", None)
+        if rca:
+            try:
+                with open(rca.get(u"data-file", ""), u"r") as rca_file:
+                    rca_data = load(rca_file, Loader=FullLoader)
+                header.insert(0, rca.get(u"title", "RCA"))
+            except (YAMLError, IOError) as err:
+                logging.warning(repr(err))
+
         if table[u"include-tests"] == u"MRR":
             hdr_param = u"Rec Rate"
         else:
@@ -909,16 +928,14 @@ def table_perf_comparison_nic(table, input_data):
                 u"Stdev of delta [%]"
             ]
         )
-        header_str = u",".join(header) + u"\n"
+        header_str = u";".join(header) + u"\n"
     except (AttributeError, KeyError) as err:
         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
         return
 
     # Prepare data to the table:
     tbl_dict = dict()
-    # topo = u""
     for job, builds in table[u"reference"][u"data"].items():
-        # topo = u"2n-skx" if u"2n-skx" in job else u""
         for build in builds:
             for tst_name, tst_data in data[job][str(build)].items():
                 if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
@@ -1076,7 +1093,6 @@ def table_perf_comparison_nic(table, input_data):
                         pass
 
     tbl_lst = list()
-    footnote = False
     for tst_name in tbl_dict:
         item = [tbl_dict[tst_name][u"name"], ]
         if history:
@@ -1114,9 +1130,6 @@ def table_perf_comparison_nic(table, input_data):
         elif item[-4] == u"Not tested":
             item.append(u"New in CSIT-2001")
             item.append(u"New in CSIT-2001")
-        # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
-        #     item.append(u"See footnote [1]")
-        #     footnote = True
         elif data_r_mean and data_c_mean:
             delta, d_stdev = relative_change_stdev(
                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
@@ -1129,6 +1142,8 @@ def table_perf_comparison_nic(table, input_data):
                 item.append(round(d_stdev))
             except ValueError:
                 item.append(d_stdev)
+        if rca_data:
+            item.insert(0, rca_data.get(item[0], u" "))
         if (len(item) == len(header)) and (item[-4] != u"Not tested"):
             tbl_lst.append(item)
 
@@ -1139,24 +1154,16 @@ def table_perf_comparison_nic(table, input_data):
     with open(csv_file, u"wt") as file_handler:
         file_handler.write(header_str)
         for test in tbl_lst:
-            file_handler.write(u",".join([str(item) for item in test]) + u"\n")
+            file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 
     txt_file_name = f"{table[u'output-file']}.txt"
-    convert_csv_to_pretty_txt(csv_file, txt_file_name)
+    convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
 
-    if footnote:
-        with open(txt_file_name, u'a') as txt_file:
-            txt_file.writelines([
-                u"\nFootnotes:\n",
-                u"[1] CSIT-1908 changed test methodology of dot1q tests in "
-                u"2-node testbeds, dot1q encapsulation is now used on both "
-                u"links of SUT.\n",
-                u"    Previously dot1q was used only on a single link with the "
-                u"other link carrying untagged Ethernet frames. This changes "
-                u"results\n",
-                u"    in slightly lower throughput in CSIT-1908 for these "
-                u"tests. See release notes."
-            ])
+    if rca_data:
+        footnote = rca_data.get(u"footnote", "")
+        if footnote:
+            with open(txt_file_name, u'a') as txt_file:
+                txt_file.writelines(footnote)
 
     # Generate html table:
     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")

diff --git a/resources/tools/presentation/pal_utils.py b/resources/tools/presentation/pal_utils.py
index 45fd277..1da3350 100644 (file)
--- a/resources/tools/presentation/pal_utils.py
+++ b/resources/tools/presentation/pal_utils.py
@@ -297,24 +297,27 @@ def classify_anomalies(data):
     return classification, avgs
 
 
-def convert_csv_to_pretty_txt(csv_file_name, txt_file_name):
+def convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u","):
     """Convert the given csv table to pretty text table.
 
     :param csv_file_name: The path to the input csv file.
     :param txt_file_name: The path to the output pretty text file.
+    :param delimiter: Delimiter for csv file.
     :type csv_file_name: str
     :type txt_file_name: str
+    :type delimiter: str
     """
 
     txt_table = None
     with open(csv_file_name, u"rt") as csv_file:
-        csv_content = csv.reader(csv_file, delimiter=u',', quotechar=u'"')
+        csv_content = csv.reader(csv_file, delimiter=delimiter, quotechar=u'"')
         for row in csv_content:
             if txt_table is None:
                 txt_table = prettytable.PrettyTable(row)
             else:
                 txt_table.add_row(row)
         txt_table.align[u"Test case"] = u"l"
+        txt_table.align[u"RCA"] = u"l"
     if txt_table:
         with open(txt_file_name, u"wt") as txt_file:
             txt_file.write(str(txt_table))

diff --git a/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml b/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml
new file mode 100644 (file)
index 0000000..564e15a
--- /dev/null
+++ b/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml
@@ -0,0 +1,46 @@
+64b-2t1c-eth-l2patch: rc1
+64b-2t1c-eth-l2xcbase-eth-2memif-1dcr: rc4
+64b-2t1c-avf-eth-l2patch: rc1
+64b-2t1c-avf-eth-l2xcbase: rc1
+64b-2t1c-eth-l2bdbasemaclrn-eth-2memif-1dcr: rc1, rc5
+64b-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc: rc1
+64b-2t1c-dot1q-l2bdbasemaclrn-eth-2memif-1dcr: rc1
+64b-2t1c-dot1q-l2xcbase: rc1
+64b-2t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-vppip4: rc1
+64b-2t1c-eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc: rc3
+64b-2t1c-dot1q-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc: rc3
+64b-2t1c-ethip4-ip4base-eth-2memif-1dcr: rc1
+64b-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm: rc1
+64b-2t1c-eth-l2xcbase-eth-2vhostvr1024-1vm: rc2
+64b-2t1c-ethip4-ip4base-eth-2vhostvr1024-1vm: rc1
+64b-2t1c-eth-l2bdscale10kmaclrn: rc1
+64b-2t1c-eth-l2bdbasemaclrn: rc1
+64b-2t1c-eth-l2bdscale100kmaclrn: rc1
+64b-2t1c-eth-l2bdscale1mmaclrn: rc1
+78b-2t1c-ethip6-ip6base: rc1
+64b-2t1c-eth-l2xcbase: rc1
+78b-2t1c-dot1q-ip6base: rc1
+64b-2t1c-dot1q-l2bdbasemaclrn-eth-2vhostvr1024-1vm: rc2
+78b-2t1c-ethip6-ip6scale200k: rc1
+78b-2t1c-ethip6-ip6scale20k: rc1
+78b-2t1c-ethip6-ip6scale2m: rc1
+64b-2t1c-dot1q-l2bdbasemaclrn: rc1
+64b-2t1c-ethip4udp-ip4base-oacl50sf-10kflows: rc1
+64b-2t1c-ethip4udp-ip4base-oacl50sl-10kflows: rc1
+64b-2t1c-avf-eth-l2bdbasemaclrn: rc1
+64b-2t1c-ethip4-ip4scale20k: rc1
+64b-2t1c-avf-dot1q-l2bdbasemaclrn: rc1, rc3
+64b-2t1c-dot1q-ip4base: rc1
+64b-2t1c-ethip4-ip4base: rc1
+64b-2t1c-ethip4udp-ip4base-iacl50sf-10kflows: rc1
+64b-2t1c-ethip4udp-ip4base-iacl50sl-10kflows: rc1
+64b-2t1c-avf-dot1q-l2bdbasemaclrn-gbp: rc1
+64b-2t1c-ethip4-ip4scale200k: rc1
+64b-2t1c-ethip4-ip4scale2m: rc1
+64b-2t1c-avf-dot1q-ip4base: rc1
+64b-2t1c-avf-ethip4-ip4base: rc1
+64b-2t1c-avf-ethip4-ip4scale200k: rc1
+64b-2t1c-avf-ethip4-ip4scale20k: rc1
+64b-2t1c-avf-ethip4-ip4scale2m: rc1
+64b-2t1c-ethip4udp-ip4base-nat44: rc1
+footnote: "\nRoot causes:\nrc1: Description 1\nrc2: Description 2\nrc3: Description 3"

diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index ddd9706..45bef9e 100644 (file)
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -3462,6 +3462,9 @@
   title: "VPP Performance Changes 2n-skx 2t1c pdr"
   algorithm: "table_perf_comparison_nic"
   output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr"
+#  rca:
+#    title: "RCA"
+#    data-file: "rca/rca-2n-skx-2t1c.yaml"
   history:
     - title: "rls1904 x710"
       data: "vpp-performance-changes-2n-skx-h1"
@@ -3479,16 +3482,19 @@
   include-tests: "PDR" # "PDR" | "NDR" | "MRR"
   filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
   parameters:
-  - "name"
-  - "parent"
-  - "throughput"
-  - "tags"
+    - "name"
+    - "parent"
+    - "throughput"
+    - "tags"
 
 # VPP Performance Changes 2n-skx 4t2c pdr
 - type: "table"
   title: "VPP Performance Changes 2n-skx 4t2c pdr"
   algorithm: "table_perf_comparison_nic"
   output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-pdr"
+#  rca:
+#    title: "RCA"
+#    data-file: "rca/rca-2n-skx-4t2c.yaml"
   history:
     - title: "rls1904 x710"
       data: "vpp-performance-changes-2n-skx-h1"
@@ -3506,16 +3512,19 @@
   include-tests: "PDR" # "PDR" | "NDR" | "MRR"
   filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
   parameters:
-  - "name"
-  - "parent"
-  - "throughput"
-  - "tags"
+    - "name"
+    - "parent"
+    - "throughput"
+    - "tags"
 
 # VPP Performance Changes 2n-skx 2t1c ndr
 - type: "table"
   title: "VPP Performance Changes 2n-skx 2t1c ndr"
   algorithm: "table_perf_comparison_nic"
   output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr"
+#  rca:
+#    title: "RCA"
+#    data-file: "rca/rca-2n-skx-2t1c.yaml"
   history:
     - title: "rls1904 x710"
       data: "vpp-performance-changes-2n-skx-h1"
@@ -3533,16 +3542,19 @@
   include-tests: "NDR" # "PDR" | "NDR" | "MRR"
   filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
   parameters:
-  - "name"
-  - "parent"
-  - "throughput"
-  - "tags"
+    - "name"
+    - "parent"
+    - "throughput"
+    - "tags"
 
 # VPP Performance Changes 2n-skx 4t2c ndr
 - type: "table"
   title: "VPP Performance Changes 2n-skx 4t2c ndr"
   algorithm: "table_perf_comparison_nic"
   output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr"
+#  rca:
+#    title: "RCA"
+#    data-file: "rca/rca-2n-skx-4t2c.yaml"
   history:
     - title: "rls1904 x710"
       data: "vpp-performance-changes-2n-skx-h1"
@@ -3560,10 +3572,10 @@
   include-tests: "NDR" # "PDR" | "NDR" | "MRR"
   filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
   parameters:
-  - "name"
-  - "parent"
-  - "throughput"
-  - "tags"
+    - "name"
+    - "parent"
+    - "throughput"
+    - "tags"
 
 ## VPP Performance Changes NFV Service Density 2n-skx 2t1c pdr
 #- type: "table"