Code Review / csit.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | review | tree
raw | inline | side by side
Report: latency comparison
[csit.git] / resources / tools / presentation / generator_tables.py
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 2644eb0..351250a 100644 (file)
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -17,6 +17,7 @@
 
 import logging
 import csv
 
 import logging
 import csv
+import math
 import re
 
 from collections import OrderedDict
 import re
 
 from collections import OrderedDict
@@ -24,6 +25,7 @@ from xml.etree import ElementTree as ET
 from datetime import datetime as dt
 from datetime import timedelta
 from copy import deepcopy
 from datetime import datetime as dt
 from datetime import timedelta
 from copy import deepcopy
+from json import loads
 
 import plotly.graph_objects as go
 import plotly.offline as ploff
 
 import plotly.graph_objects as go
 import plotly.offline as ploff
@@ -93,7 +95,7 @@ def table_oper_data_html(table, input_data):
     )
     data = input_data.filter_data(
         table,
     )
     data = input_data.filter_data(
         table,
-        params=[u"name", u"parent", u"show-run", u"type"],
+        params=[u"name", u"parent", u"telemetry-show-run", u"type"],
         continue_on_error=True
     )
     if data.empty:
         continue_on_error=True
     )
     if data.empty:
@@ -146,7 +148,8 @@ def table_oper_data_html(table, input_data):
         )
         thead.text = u"\t"
 
         )
         thead.text = u"\t"
 
-        if tst_data.get(u"show-run", u"No Data") == u"No Data":
+        if tst_data.get(u"telemetry-show-run", None) is None or \
+                isinstance(tst_data[u"telemetry-show-run"], str):
             trow = ET.SubElement(
                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
             )
             trow = ET.SubElement(
                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
             )
@@ -176,17 +179,56 @@ def table_oper_data_html(table, input_data):
             u"Average Vector Size"
         )
 
             u"Average Vector Size"
         )
 
-        for dut_data in tst_data[u"show-run"].values():
+        for dut_data in tst_data[u"telemetry-show-run"].values():
             trow = ET.SubElement(
                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
             )
             tcol = ET.SubElement(
                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
             )
             trow = ET.SubElement(
                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
             )
             tcol = ET.SubElement(
                 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
             )
-            if dut_data.get(u"threads", None) is None:
+            if dut_data.get(u"runtime", None) is None:
                 tcol.text = u"No Data"
                 continue
 
                 tcol.text = u"No Data"
                 continue
 
+            runtime = dict()
+            for item in dut_data[u"runtime"].get(u"data", tuple()):
+                tid = int(item[u"labels"][u"thread_id"])
+                if runtime.get(tid, None) is None:
+                    runtime[tid] = dict()
+                gnode = item[u"labels"][u"graph_node"]
+                if runtime[tid].get(gnode, None) is None:
+                    runtime[tid][gnode] = dict()
+                try:
+                    runtime[tid][gnode][item[u"name"]] = float(item[u"value"])
+                except ValueError:
+                    runtime[tid][gnode][item[u"name"]] = item[u"value"]
+
+            threads = dict({idx: list() for idx in range(len(runtime))})
+            for idx, run_data in runtime.items():
+                for gnode, gdata in run_data.items():
+                    if gdata[u"vectors"] > 0:
+                        clocks = gdata[u"clocks"] / gdata[u"vectors"]
+                    elif gdata[u"calls"] > 0:
+                        clocks = gdata[u"clocks"] / gdata[u"calls"]
+                    elif gdata[u"suspends"] > 0:
+                        clocks = gdata[u"clocks"] / gdata[u"suspends"]
+                    else:
+                        clocks = 0.0
+                    if gdata[u"calls"] > 0:
+                        vectors_call = gdata[u"vectors"] / gdata[u"calls"]
+                    else:
+                        vectors_call = 0.0
+                    if int(gdata[u"calls"]) + int(gdata[u"vectors"]) + \
+                            int(gdata[u"suspends"]):
+                        threads[idx].append([
+                            gnode,
+                            int(gdata[u"calls"]),
+                            int(gdata[u"vectors"]),
+                            int(gdata[u"suspends"]),
+                            clocks,
+                            vectors_call
+                        ])
+
             bold = ET.SubElement(tcol, u"b")
             bold.text = (
                 f"Host IP: {dut_data.get(u'host', '')}, "
             bold = ET.SubElement(tcol, u"b")
             bold.text = (
                 f"Host IP: {dut_data.get(u'host', '')}, "
@@ -200,7 +242,7 @@ def table_oper_data_html(table, input_data):
             )
             thead.text = u"\t"
 
             )
             thead.text = u"\t"
 
-            for thread_nr, thread in dut_data[u"threads"].items():
+            for thread_nr, thread in threads.items():
                 trow = ET.SubElement(
                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
                 )
                 trow = ET.SubElement(
                     tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
                 )
@@ -323,7 +365,8 @@ def table_merged_details(table, input_data):
         suite_name = suite[u"name"]
         table_lst = list()
         for test in data.keys():
         suite_name = suite[u"name"]
         table_lst = list()
         for test in data.keys():
-            if data[test][u"parent"] not in suite_name:
+            if data[test][u"status"] != u"PASS" or \
+                    data[test][u"parent"] not in suite_name:
                 continue
             row_lst = list()
             for column in table[u"columns"]:
                 continue
             row_lst = list()
             for column in table[u"columns"]:
@@ -348,13 +391,14 @@ def table_merged_details(table, input_data):
                         # Temporary solution: remove NDR results from message:
                         if bool(table.get(u'remove-ndr', False)):
                             try:
                         # Temporary solution: remove NDR results from message:
                         if bool(table.get(u'remove-ndr', False)):
                             try:
-                                col_data = col_data.split(u" |br| ", 1)[1]
+                                col_data = col_data.split(u"\n", 1)[1]
                             except IndexError:
                                 pass
                             except IndexError:
                                 pass
+                        col_data = col_data.replace(u'\n', u' |br| ').\
+                            replace(u'\r', u'').replace(u'"', u"'")
                         col_data = f" |prein| {col_data} |preout| "
                         col_data = f" |prein| {col_data} |preout| "
-                    elif column[u"data"].split(u" ")[1] in \
-                            (u"conf-history", u"show-run"):
-                        col_data = col_data.replace(u" |br| ", u"", 1)
+                    elif column[u"data"].split(u" ")[1] in (u"conf-history", ):
+                        col_data = col_data.replace(u'\n', u' |br| ')
                         col_data = f" |prein| {col_data[:-5]} |preout| "
                     row_lst.append(f'"{col_data}"')
                 except KeyError:
                         col_data = f" |prein| {col_data[:-5]} |preout| "
                     row_lst.append(f'"{col_data}"')
                 except KeyError:
@@ -386,12 +430,7 @@ def _tpc_modify_test_name(test_name, ignore_nic=False):
     :rtype: str
     """
     test_name_mod = test_name.\
     :rtype: str
     """
     test_name_mod = test_name.\
-        replace(u"-ndrpdrdisc", u""). \
         replace(u"-ndrpdr", u"").\
         replace(u"-ndrpdr", u"").\
-        replace(u"-pdrdisc", u""). \
-        replace(u"-ndrdisc", u"").\
-        replace(u"-pdr", u""). \
-        replace(u"-ndr", u""). \
         replace(u"1t1c", u"1c").\
         replace(u"2t1c", u"1c"). \
         replace(u"2t2c", u"2c").\
         replace(u"1t1c", u"1c").\
         replace(u"2t1c", u"1c"). \
         replace(u"2t2c", u"2c").\
@@ -425,7 +464,7 @@ def _tpc_insert_data(target, src, include_tests):
     """Insert src data to the target structure.
 
     :param target: Target structure where the data is placed.
     """Insert src data to the target structure.
 
     :param target: Target structure where the data is placed.
-    :param src: Source data to be placed into the target stucture.
+    :param src: Source data to be placed into the target structure.
     :param include_tests: Which results will be included (MRR, NDR, PDR).
     :type target: list
     :type src: dict
     :param include_tests: Which results will be included (MRR, NDR, PDR).
     :type target: list
     :type src: dict
@@ -439,6 +478,13 @@ def _tpc_insert_data(target, src, include_tests):
             target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
         elif include_tests == u"NDR":
             target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
             target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
         elif include_tests == u"NDR":
             target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
+        elif u"latency" in include_tests:
+            keys = include_tests.split(u"-")
+            if len(keys) == 4:
+                lat = src[keys[0]][keys[1]][keys[2]][keys[3]]
+                target[u"data"].append(
+                    float(u"nan") if lat == -1 else lat * 1e6
+                )
     except (KeyError, TypeError):
         pass
 
     except (KeyError, TypeError):
         pass
 
@@ -594,6 +640,7 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
         path = u"_tmp/src/vpp_performance_tests/comparisons/"
     else:
         path = u"_tmp/src/dpdk_performance_tests/comparisons/"
         path = u"_tmp/src/vpp_performance_tests/comparisons/"
     else:
         path = u"_tmp/src/dpdk_performance_tests/comparisons/"
+    logging.info(f"    Writing the HTML file to {path}{file_name}.rst")
     with open(f"{path}{file_name}.rst", u"wt") as rst_file:
         rst_file.write(
             u"\n"
     with open(f"{path}{file_name}.rst", u"wt") as rst_file:
         rst_file.write(
             u"\n"
@@ -612,11 +659,22 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
             f'</iframe>\n\n'
         )
 
             f'</iframe>\n\n'
         )
 
-        # TODO: Use html (rst) list for legend and footnote
         if legend:
         if legend:
-            rst_file.write(legend[1:].replace(u"\n", u" |br| "))
+            try:
+                itm_lst = legend[1:-2].split(u"\n")
+                rst_file.write(
+                    f"{itm_lst[0]}\n\n- " + u'\n- '.join(itm_lst[1:]) + u"\n\n"
+                )
+            except IndexError as err:
+                logging.error(f"Legend cannot be written to html file\n{err}")
         if footnote:
         if footnote:
-            rst_file.write(footnote.replace(u"\n", u" |br| ")[1:])
+            try:
+                itm_lst = footnote[1:].split(u"\n")
+                rst_file.write(
+                    f"{itm_lst[0]}\n\n- " + u'\n- '.join(itm_lst[1:]) + u"\n\n"
+                )
+            except IndexError as err:
+                logging.error(f"Footnote cannot be written to html file\n{err}")
 
 
 def table_soak_vs_ndr(table, input_data):
 
 
 def table_soak_vs_ndr(table, input_data):
@@ -777,17 +835,17 @@ def table_soak_vs_ndr(table, input_data):
     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 
     # Generate csv tables:
     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 
     # Generate csv tables:
-    csv_file = f"{table[u'output-file']}.csv"
-    with open(csv_file, u"wt") as file_handler:
+    csv_file_name = f"{table[u'output-file']}.csv"
+    with open(csv_file_name, u"wt") as file_handler:
         file_handler.write(header_str)
         for test in tbl_lst:
             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 
     convert_csv_to_pretty_txt(
         file_handler.write(header_str)
         for test in tbl_lst:
             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 
     convert_csv_to_pretty_txt(
-        csv_file, f"{table[u'output-file']}.txt", delimiter=u";"
+        csv_file_name, f"{table[u'output-file']}.txt", delimiter=u";"
     )
     )
-    with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
-        txt_file.write(legend)
+    with open(f"{table[u'output-file']}.txt", u'a') as file_handler:
+        file_handler.write(legend)
 
     # Generate html table:
     _tpc_generate_html_table(
 
     # Generate html table:
     _tpc_generate_html_table(
@@ -867,7 +925,11 @@ def table_perf_trending_dash(table, input_data):
         if len(data_t) < 2:
             continue
 
         if len(data_t) < 2:
             continue
 
-        classification_lst, avgs, _ = classify_anomalies(data_t)
+        try:
+            classification_lst, avgs, _ = classify_anomalies(data_t)
+        except ValueError as err:
+            logging.info(f"{err} Skipping")
+            return
 
         win_size = min(len(data_t), table[u"window"])
         long_win_size = min(len(data_t), table[u"long-trend-window"])
 
         win_size = min(len(data_t), table[u"window"])
         long_win_size = min(len(data_t), table[u"long-trend-window"])
@@ -908,13 +970,14 @@ def table_perf_trending_dash(table, input_data):
                  classification_lst[-win_size+1:].count(u"progression")])
 
     tbl_lst.sort(key=lambda rel: rel[0])
                  classification_lst[-win_size+1:].count(u"progression")])
 
     tbl_lst.sort(key=lambda rel: rel[0])
+    tbl_lst.sort(key=lambda rel: rel[3])
+    tbl_lst.sort(key=lambda rel: rel[2])
 
     tbl_sorted = list()
     for nrr in range(table[u"window"], -1, -1):
         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
         for nrp in range(table[u"window"], -1, -1):
             tbl_out = [item for item in tbl_reg if item[5] == nrp]
 
     tbl_sorted = list()
     for nrr in range(table[u"window"], -1, -1):
         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
         for nrp in range(table[u"window"], -1, -1):
             tbl_out = [item for item in tbl_reg if item[5] == nrp]
-            tbl_out.sort(key=lambda rel: rel[2])
             tbl_sorted.extend(tbl_out)
 
     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
             tbl_sorted.extend(tbl_out)
 
     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
@@ -977,27 +1040,27 @@ def _generate_url(testbed, test_name):
 
     if u"1t1c" in test_name or \
         (u"-1c-" in test_name and
 
     if u"1t1c" in test_name or \
         (u"-1c-" in test_name and
-         testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
+         testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv", u"2n-tx2")):
         cores = u"1t1c"
     elif u"2t2c" in test_name or \
          (u"-2c-" in test_name and
         cores = u"1t1c"
     elif u"2t2c" in test_name or \
          (u"-2c-" in test_name and
-          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
+          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv", u"2n-tx2")):
         cores = u"2t2c"
     elif u"4t4c" in test_name or \
          (u"-4c-" in test_name and
         cores = u"2t2c"
     elif u"4t4c" in test_name or \
          (u"-4c-" in test_name and
-          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
+          testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv", u"2n-tx2")):
         cores = u"4t4c"
     elif u"2t1c" in test_name or \
          (u"-1c-" in test_name and
         cores = u"4t4c"
     elif u"2t1c" in test_name or \
          (u"-1c-" in test_name and
-          testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
+          testbed in (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2")):
         cores = u"2t1c"
     elif u"4t2c" in test_name or \
          (u"-2c-" in test_name and
         cores = u"2t1c"
     elif u"4t2c" in test_name or \
          (u"-2c-" in test_name and
-          testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
+          testbed in (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2")):
         cores = u"4t2c"
     elif u"8t4c" in test_name or \
          (u"-4c-" in test_name and
         cores = u"4t2c"
     elif u"8t4c" in test_name or \
          (u"-4c-" in test_name and
-          testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
+          testbed in (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2")):
         cores = u"8t4c"
     else:
         cores = u""
         cores = u"8t4c"
     else:
         cores = u""
@@ -1015,11 +1078,53 @@ def _generate_url(testbed, test_name):
     else:
         driver = u"dpdk"
 
     else:
         driver = u"dpdk"
 
-    if u"acl" in test_name or \
-            u"macip" in test_name or \
-            u"nat" in test_name or \
-            u"policer" in test_name or \
-            u"cop" in test_name:
+    if u"macip-iacl1s" in test_name:
+        bsf = u"features-macip-iacl1"
+    elif u"macip-iacl10s" in test_name:
+        bsf = u"features-macip-iacl10"
+    elif u"macip-iacl50s" in test_name:
+        bsf = u"features-macip-iacl50"
+    elif u"iacl1s" in test_name:
+        bsf = u"features-iacl1"
+    elif u"iacl10s" in test_name:
+        bsf = u"features-iacl10"
+    elif u"iacl50s" in test_name:
+        bsf = u"features-iacl50"
+    elif u"oacl1s" in test_name:
+        bsf = u"features-oacl1"
+    elif u"oacl10s" in test_name:
+        bsf = u"features-oacl10"
+    elif u"oacl50s" in test_name:
+        bsf = u"features-oacl50"
+    elif u"nat44det" in test_name:
+        bsf = u"nat44det-bidir"
+    elif u"nat44ed" in test_name and u"udir" in test_name:
+        bsf = u"nat44ed-udir"
+    elif u"-cps" in test_name and u"ethip4udp" in test_name:
+        bsf = u"udp-cps"
+    elif u"-cps" in test_name and u"ethip4tcp" in test_name:
+        bsf = u"tcp-cps"
+    elif u"-pps" in test_name and u"ethip4udp" in test_name:
+        bsf = u"udp-pps"
+    elif u"-pps" in test_name and u"ethip4tcp" in test_name:
+        bsf = u"tcp-pps"
+    elif u"-tput" in test_name and u"ethip4udp" in test_name:
+        bsf = u"udp-tput"
+    elif u"-tput" in test_name and u"ethip4tcp" in test_name:
+        bsf = u"tcp-tput"
+    elif u"udpsrcscale" in test_name:
+        bsf = u"features-udp"
+    elif u"iacl" in test_name:
+        bsf = u"features"
+    elif u"policer" in test_name:
+        bsf = u"features"
+    elif u"adl" in test_name:
+        bsf = u"features"
+    elif u"cop" in test_name:
+        bsf = u"features"
+    elif u"nat" in test_name:
+        bsf = u"features"
+    elif u"macip" in test_name:
         bsf = u"features"
     elif u"scale" in test_name:
         bsf = u"scale"
         bsf = u"features"
     elif u"scale" in test_name:
         bsf = u"scale"
@@ -1030,6 +1135,24 @@ def _generate_url(testbed, test_name):
 
     if u"114b" in test_name and u"vhost" in test_name:
         domain = u"vts"
 
     if u"114b" in test_name and u"vhost" in test_name:
         domain = u"vts"
+    elif u"nat44" in test_name or u"-pps" in test_name or u"-cps" in test_name:
+        domain = u"nat44"
+        if u"nat44det" in test_name:
+            domain += u"-det-bidir"
+        else:
+            domain += u"-ed"
+        if u"udir" in test_name:
+            domain += u"-unidir"
+        elif u"-ethip4udp-" in test_name:
+            domain += u"-udp"
+        elif u"-ethip4tcp-" in test_name:
+            domain += u"-tcp"
+        if u"-cps" in test_name:
+            domain += u"-cps"
+        elif u"-pps" in test_name:
+            domain += u"-pps"
+        elif u"-tput" in test_name:
+            domain += u"-tput"
     elif u"testpmd" in test_name or u"l3fwd" in test_name:
         domain = u"dpdk"
     elif u"memif" in test_name:
     elif u"testpmd" in test_name or u"l3fwd" in test_name:
         domain = u"dpdk"
     elif u"memif" in test_name:
@@ -1058,6 +1181,8 @@ def _generate_url(testbed, test_name):
             bsf += u"-hw"
     elif u"ethip4vxlan" in test_name:
         domain = u"ip4_tunnels"
             bsf += u"-hw"
     elif u"ethip4vxlan" in test_name:
         domain = u"ip4_tunnels"
+    elif u"ethip4udpgeneve" in test_name:
+        domain = u"ip4_tunnels"
     elif u"ip4base" in test_name or u"ip4scale" in test_name:
         domain = u"ip4"
     elif u"ip6base" in test_name or u"ip6scale" in test_name:
     elif u"ip4base" in test_name or u"ip4scale" in test_name:
         domain = u"ip4"
     elif u"ip6base" in test_name or u"ip6scale" in test_name:
@@ -1093,15 +1218,33 @@ def table_perf_trending_dash_html(table, input_data):
     if not table.get(u"testbed", None):
         logging.error(
             f"The testbed is not defined for the table "
     if not table.get(u"testbed", None):
         logging.error(
             f"The testbed is not defined for the table "
-            f"{table.get(u'title', u'')}."
+            f"{table.get(u'title', u'')}. Skipping."
         )
         return
 
         )
         return
 
+    test_type = table.get(u"test-type", u"MRR")
+    if test_type not in (u"MRR", u"NDR", u"PDR"):
+        logging.error(
+            f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
+            f"Skipping."
+        )
+        return
+
+    if test_type in (u"NDR", u"PDR"):
+        lnk_dir = u"../ndrpdr_trending/"
+        lnk_sufix = f"-{test_type.lower()}"
+    else:
+        lnk_dir = u"../trending/"
+        lnk_sufix = u""
+
     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 
     try:
         with open(table[u"input-file"], u'rt') as csv_file:
             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 
     try:
         with open(table[u"input-file"], u'rt') as csv_file:
             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
+    except FileNotFoundError as err:
+        logging.warning(f"{err}")
+        return
     except KeyError:
         logging.warning(u"The input file is not defined.")
         return
     except KeyError:
         logging.warning(u"The input file is not defined.")
         return
@@ -1161,8 +1304,9 @@ def table_perf_trending_dash_html(table, input_data):
                     tdata,
                     u"a",
                     attrib=dict(
                     tdata,
                     u"a",
                     attrib=dict(
-                        href=f"../trending/"
-                             f"{_generate_url(table.get(u'testbed', ''), item)}"
+                        href=f"{lnk_dir}"
+                        f"{_generate_url(table.get(u'testbed', ''), item)}"
+                        f"{lnk_sufix}"
                     )
                 )
                 ref.text = item
                     )
                 )
                 ref.text = item
@@ -1212,6 +1356,8 @@ def table_last_failed_tests(table, input_data):
             build = str(build)
             try:
                 version = input_data.metadata(job, build).get(u"version", u"")
             build = str(build)
             try:
                 version = input_data.metadata(job, build).get(u"version", u"")
+                duration = \
+                    input_data.metadata(job, build).get(u"elapsedtime", u"")
             except KeyError:
                 logging.error(f"Data for {job}: {build} is not present.")
                 return
             except KeyError:
                 logging.error(f"Data for {job}: {build} is not present.")
                 return
@@ -1230,15 +1376,16 @@ def table_last_failed_tests(table, input_data):
                     continue
                 nic = groups.group(0)
                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
                     continue
                 nic = groups.group(0)
                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
-            tbl_list.append(str(passed))
-            tbl_list.append(str(failed))
+            tbl_list.append(passed)
+            tbl_list.append(failed)
+            tbl_list.append(duration)
             tbl_list.extend(failed_tests)
 
     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
     logging.info(f"    Writing file: {file_name}")
     with open(file_name, u"wt") as file_handler:
         for test in tbl_list:
             tbl_list.extend(failed_tests)
 
     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
     logging.info(f"    Writing file: {file_name}")
     with open(file_name, u"wt") as file_handler:
         for test in tbl_list:
-            file_handler.write(test + u'\n')
+            file_handler.write(f"{test}\n")
 
 
 def table_failed_tests(table, input_data):
 
 
 def table_failed_tests(table, input_data):
@@ -1368,10 +1515,25 @@ def table_failed_tests_html(table, input_data):
     if not table.get(u"testbed", None):
         logging.error(
             f"The testbed is not defined for the table "
     if not table.get(u"testbed", None):
         logging.error(
             f"The testbed is not defined for the table "
-            f"{table.get(u'title', u'')}."
+            f"{table.get(u'title', u'')}. Skipping."
         )
         return
 
         )
         return
 
+    test_type = table.get(u"test-type", u"MRR")
+    if test_type not in (u"MRR", u"NDR", u"PDR", u"NDRPDR"):
+        logging.error(
+            f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
+            f"Skipping."
+        )
+        return
+
+    if test_type in (u"NDRPDR", u"NDR", u"PDR"):
+        lnk_dir = u"../ndrpdr_trending/"
+        lnk_sufix = u"-pdr"
+    else:
+        lnk_dir = u"../trending/"
+        lnk_sufix = u""
+
     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 
     try:
     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
 
     try:
@@ -1413,13 +1575,14 @@ def table_failed_tests_html(table, input_data):
                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
             )
             # Name:
                 attrib=dict(align=u"left" if c_idx == 0 else u"center")
             )
             # Name:
-            if c_idx == 0:
+            if c_idx == 0 and table.get(u"add-links", True):
                 ref = ET.SubElement(
                     tdata,
                     u"a",
                     attrib=dict(
                 ref = ET.SubElement(
                     tdata,
                     u"a",
                     attrib=dict(
-                        href=f"../trending/"
-                             f"{_generate_url(table.get(u'testbed', ''), item)}"
+                        href=f"{lnk_dir}"
+                        f"{_generate_url(table.get(u'testbed', ''), item)}"
+                        f"{lnk_sufix}"
                     )
                 )
                 ref.text = item
                     )
                 )
                 ref.text = item
@@ -1468,7 +1631,14 @@ def table_comparison(table, input_data):
         tag = col.get(u"tag", None)
         data = input_data.filter_data(
             table,
         tag = col.get(u"tag", None)
         data = input_data.filter_data(
             table,
-            params=[u"throughput", u"result", u"name", u"parent", u"tags"],
+            params=[
+                u"throughput",
+                u"result",
+                u"latency",
+                u"name",
+                u"parent",
+                u"tags"
+            ],
             data=col[u"data-set"],
             continue_on_error=True
         )
             data=col[u"data-set"],
             continue_on_error=True
         )
@@ -1506,7 +1676,14 @@ def table_comparison(table, input_data):
         if replacement:
             rpl_data = input_data.filter_data(
                 table,
         if replacement:
             rpl_data = input_data.filter_data(
                 table,
-                params=[u"throughput", u"result", u"name", u"parent", u"tags"],
+                params=[
+                    u"throughput",
+                    u"result",
+                    u"latency",
+                    u"name",
+                    u"parent",
+                    u"tags"
+                ],
                 data=replacement,
                 continue_on_error=True
             )
                 data=replacement,
                 continue_on_error=True
             )
@@ -1540,7 +1717,8 @@ def table_comparison(table, input_data):
                             include_tests=table[u"include-tests"]
                         )
 
                             include_tests=table[u"include-tests"]
                         )
 
-        if table[u"include-tests"] in (u"NDR", u"PDR"):
+        if table[u"include-tests"] in (u"NDR", u"PDR") or \
+                u"latency" in table[u"include-tests"]:
             for tst_name, tst_data in col_data[u"data"].items():
                 if tst_data[u"data"]:
                     tst_data[u"mean"] = mean(tst_data[u"data"])
             for tst_name, tst_data in col_data[u"data"].items():
                 if tst_data[u"data"]:
                     tst_data[u"mean"] = mean(tst_data[u"data"])
@@ -1572,6 +1750,7 @@ def table_comparison(table, input_data):
         tbl_lst.append(row)
 
     comparisons = table.get(u"comparisons", None)
         tbl_lst.append(row)
 
     comparisons = table.get(u"comparisons", None)
+    rcas = list()
     if comparisons and isinstance(comparisons, list):
         for idx, comp in enumerate(comparisons):
             try:
     if comparisons and isinstance(comparisons, list):
         for idx, comp in enumerate(comparisons):
             try:
@@ -1581,19 +1760,38 @@ def table_comparison(table, input_data):
                 logging.warning(u"Comparison: No references defined! Skipping.")
                 comparisons.pop(idx)
                 continue
                 logging.warning(u"Comparison: No references defined! Skipping.")
                 comparisons.pop(idx)
                 continue
-            if not (0 < col_ref <= len(cols) and
-                    0 < col_cmp <= len(cols)) or \
-                    col_ref == col_cmp:
+            if not (0 < col_ref <= len(cols) and 0 < col_cmp <= len(cols) or
+                    col_ref == col_cmp):
                 logging.warning(f"Wrong values of reference={col_ref} "
                                 f"and/or compare={col_cmp}. Skipping.")
                 comparisons.pop(idx)
                 continue
                 logging.warning(f"Wrong values of reference={col_ref} "
                                 f"and/or compare={col_cmp}. Skipping.")
                 comparisons.pop(idx)
                 continue
+            rca_file_name = comp.get(u"rca-file", None)
+            if rca_file_name:
+                try:
+                    with open(rca_file_name, u"r") as file_handler:
+                        rcas.append(
+                            {
+                                u"title": f"RCA{idx + 1}",
+                                u"data": load(file_handler, Loader=FullLoader)
+                            }
+                        )
+                except (YAMLError, IOError) as err:
+                    logging.warning(
+                        f"The RCA file {rca_file_name} does not exist or "
+                        f"it is corrupted!"
+                    )
+                    logging.debug(repr(err))
+                    rcas.append(None)
+            else:
+                rcas.append(None)
+    else:
+        comparisons = None
 
     tbl_cmp_lst = list()
     if comparisons:
         for row in tbl_lst:
             new_row = deepcopy(row)
 
     tbl_cmp_lst = list()
     if comparisons:
         for row in tbl_lst:
             new_row = deepcopy(row)
-            add_to_tbl = False
             for comp in comparisons:
                 ref_itm = row[int(comp[u"reference"])]
                 if ref_itm is None and \
             for comp in comparisons:
                 ref_itm = row[int(comp[u"reference"])]
                 if ref_itm is None and \
@@ -1605,55 +1803,45 @@ def table_comparison(table, input_data):
                         cmp_itm[u"mean"] is not None and \
                         ref_itm[u"stdev"] is not None and \
                         cmp_itm[u"stdev"] is not None:
                         cmp_itm[u"mean"] is not None and \
                         ref_itm[u"stdev"] is not None and \
                         cmp_itm[u"stdev"] is not None:
-                    delta, d_stdev = relative_change_stdev(
-                        ref_itm[u"mean"], cmp_itm[u"mean"],
-                        ref_itm[u"stdev"], cmp_itm[u"stdev"]
-                    )
-                    new_row.append(
-                        {
-                            u"mean": delta * 1e6,
-                            u"stdev": d_stdev * 1e6
-                        }
-                    )
-                    add_to_tbl = True
+                    try:
+                        delta, d_stdev = relative_change_stdev(
+                            ref_itm[u"mean"], cmp_itm[u"mean"],
+                            ref_itm[u"stdev"], cmp_itm[u"stdev"]
+                        )
+                    except ZeroDivisionError:
+                        break
+                    if delta is None or math.isnan(delta):
+                        break
+                    new_row.append({
+                        u"mean": delta * 1e6,
+                        u"stdev": d_stdev * 1e6
+                    })
                 else:
                 else:
-                    new_row.append(None)
-            if add_to_tbl:
+                    break
+            else:
                 tbl_cmp_lst.append(new_row)
 
                 tbl_cmp_lst.append(new_row)
 
-    tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
-    tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
-
-    rcas = list()
-    rca_in = table.get(u"rca", None)
-    if rca_in and isinstance(rca_in, list):
-        for idx, itm in enumerate(rca_in):
-            try:
-                with open(itm.get(u"data", u""), u"r") as rca_file:
-                    rcas.append(
-                        {
-                            u"title": itm.get(u"title", f"RCA{idx}"),
-                            u"data": load(rca_file, Loader=FullLoader)
-                        }
-                    )
-            except (YAMLError, IOError) as err:
-                logging.warning(
-                    f"The RCA file {itm.get(u'data', u'')} does not exist or "
-                    f"it is corrupted!"
-                )
-                logging.debug(repr(err))
+    try:
+        tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
+        tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
+    except TypeError as err:
+        logging.warning(f"Empty data element in table\n{tbl_cmp_lst}\n{err}")
 
     tbl_for_csv = list()
     for line in tbl_cmp_lst:
         row = [line[0], ]
         for idx, itm in enumerate(line[1:]):
 
     tbl_for_csv = list()
     for line in tbl_cmp_lst:
         row = [line[0], ]
         for idx, itm in enumerate(line[1:]):
-            if itm is None:
+            if itm is None or not isinstance(itm, dict) or\
+                    itm.get(u'mean', None) is None or \
+                    itm.get(u'stdev', None) is None:
                 row.append(u"NT")
                 row.append(u"NT")
             else:
                 row.append(round(float(itm[u'mean']) / 1e6, 3))
                 row.append(round(float(itm[u'stdev']) / 1e6, 3))
         for rca in rcas:
                 row.append(u"NT")
                 row.append(u"NT")
             else:
                 row.append(round(float(itm[u'mean']) / 1e6, 3))
                 row.append(round(float(itm[u'stdev']) / 1e6, 3))
         for rca in rcas:
+            if rca is None:
+                continue
             rca_nr = rca[u"data"].get(row[0], u"-")
             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
         tbl_for_csv.append(row)
             rca_nr = rca[u"data"].get(row[0], u"-")
             row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
         tbl_for_csv.append(row)
@@ -1669,7 +1857,9 @@ def table_comparison(table, input_data):
         header_csv.append(
             f"Stdev({comp.get(u'title', u'')})"
         )
         header_csv.append(
             f"Stdev({comp.get(u'title', u'')})"
         )
-    header_csv.extend([rca[u"title"] for rca in rcas])
+    for rca in rcas:
+        if rca:
+            header_csv.append(rca[u"title"])
 
     legend_lst = table.get(u"legend", None)
     if legend_lst is None:
 
     legend_lst = table.get(u"legend", None)
     if legend_lst is None:
@@ -1678,12 +1868,14 @@ def table_comparison(table, input_data):
         legend = u"\n" + u"\n".join(legend_lst) + u"\n"
 
     footnote = u""
         legend = u"\n" + u"\n".join(legend_lst) + u"\n"
 
     footnote = u""
-    for rca in rcas:
-        footnote += f"\n{rca[u'title']}:\n"
-        footnote += rca[u"data"].get(u"footnote", u"")
+    if rcas and any(rcas):
+        footnote += u"\nRoot Cause Analysis:\n"
+        for rca in rcas:
+            if rca:
+                footnote += f"{rca[u'data'].get(u'footnote', u'')}\n"
 
 
-    csv_file = f"{table[u'output-file']}-csv.csv"
-    with open(csv_file, u"wt", encoding='utf-8') as file_handler:
+    csv_file_name = f"{table[u'output-file']}-csv.csv"
+    with open(csv_file_name, u"wt", encoding='utf-8') as file_handler:
         file_handler.write(
             u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
         )
         file_handler.write(
             u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
         )
@@ -1703,7 +1895,9 @@ def table_comparison(table, input_data):
     for line in tbl_cmp_lst:
         row = [line[0], ]
         for idx, itm in enumerate(line[1:]):
     for line in tbl_cmp_lst:
         row = [line[0], ]
         for idx, itm in enumerate(line[1:]):
-            if itm is None:
+            if itm is None or not isinstance(itm, dict) or \
+                    itm.get(u'mean', None) is None or \
+                    itm.get(u'stdev', None) is None:
                 new_itm = u"NT"
             else:
                 if idx < len(cols):
                 new_itm = u"NT"
             else:
                 if idx < len(cols):
@@ -1724,6 +1918,10 @@ def table_comparison(table, input_data):
 
         tbl_tmp.append(row)
 
 
         tbl_tmp.append(row)
 
+    header = [u"Test Case", ]
+    header.extend([col[u"title"] for col in cols])
+    header.extend([comp.get(u"title", u"") for comp in comparisons])
+
     tbl_final = list()
     for line in tbl_tmp:
         row = [line[0], ]
     tbl_final = list()
     for line in tbl_tmp:
         row = [line[0], ]
@@ -1734,32 +1932,43 @@ def table_comparison(table, input_data):
             itm_lst = itm.rsplit(u"\u00B1", 1)
             itm_lst[-1] = \
                 f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
             itm_lst = itm.rsplit(u"\u00B1", 1)
             itm_lst[-1] = \
                 f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
-            row.append(u"\u00B1".join(itm_lst))
-        for rca in rcas:
-            rca_nr = rca[u"data"].get(row[0], u"-")
-            row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
-
+            itm_str = u"\u00B1".join(itm_lst)
+
+            if idx >= len(cols):
+                # Diffs
+                rca = rcas[idx - len(cols)]
+                if rca:
+                    # Add rcas to diffs
+                    rca_nr = rca[u"data"].get(row[0], None)
+                    if rca_nr:
+                        hdr_len = len(header[idx + 1]) - 1
+                        if hdr_len < 19:
+                            hdr_len = 19
+                        rca_nr = f"[{rca_nr}]"
+                        itm_str = (
+                            f"{u' ' * (4 - len(rca_nr))}{rca_nr}"
+                            f"{u' ' * (hdr_len - 4 - len(itm_str))}"
+                            f"{itm_str}"
+                        )
+            row.append(itm_str)
         tbl_final.append(row)
 
         tbl_final.append(row)
 
-    header = [u"Test Case", ]
-    header.extend([col[u"title"] for col in cols])
-    header.extend([comp.get(u"title", u"") for comp in comparisons])
-    header.extend([rca[u"title"] for rca in rcas])
-
     # Generate csv tables:
     # Generate csv tables:
-    csv_file = f"{table[u'output-file']}.csv"
-    with open(csv_file, u"wt", encoding='utf-8') as file_handler:
+    csv_file_name = f"{table[u'output-file']}.csv"
+    logging.info(f"    Writing the file {csv_file_name}")
+    with open(csv_file_name, u"wt", encoding='utf-8') as file_handler:
         file_handler.write(u";".join(header) + u"\n")
         for test in tbl_final:
             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 
     # Generate txt table:
     txt_file_name = f"{table[u'output-file']}.txt"
         file_handler.write(u";".join(header) + u"\n")
         for test in tbl_final:
             file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 
     # Generate txt table:
     txt_file_name = f"{table[u'output-file']}.txt"
-    convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
+    logging.info(f"    Writing the file {txt_file_name}")
+    convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u";")
 
 
-    with open(txt_file_name, u'a', encoding='utf-8') as txt_file:
-        txt_file.write(legend)
-        txt_file.write(footnote)
+    with open(txt_file_name, u'a', encoding='utf-8') as file_handler:
+        file_handler.write(legend)
+        file_handler.write(footnote)
 
     # Generate html table:
     _tpc_generate_html_table(
 
     # Generate html table:
     _tpc_generate_html_table(
@@ -1875,12 +2084,13 @@ def table_weekly_comparison(table, in_data):
             ref_data = tst_data.get(idx_ref, None)
             cmp_data = tst_data.get(idx_cmp, None)
             if ref_data is None or cmp_data is None:
             ref_data = tst_data.get(idx_ref, None)
             cmp_data = tst_data.get(idx_cmp, None)
             if ref_data is None or cmp_data is None:
-                cmp_dict[tst_name].append(float('nan'))
+                cmp_dict[tst_name].append(float(u'nan'))
             else:
                 cmp_dict[tst_name].append(
                     relative_change(ref_data, cmp_data)
                 )
 
             else:
                 cmp_dict[tst_name].append(
                     relative_change(ref_data, cmp_data)
                 )
 
+    tbl_lst_none = list()
     tbl_lst = list()
     for tst_name, tst_data in tbl_dict.items():
         itm_lst = [tst_data[u"name"], ]
     tbl_lst = list()
     for tst_name, tst_data in tbl_dict.items():
         itm_lst = [tst_data[u"name"], ]
@@ -1896,14 +2106,20 @@ def table_weekly_comparison(table, in_data):
                 for itm in cmp_dict[tst_name]
             ]
         )
                 for itm in cmp_dict[tst_name]
             ]
         )
-        tbl_lst.append(itm_lst)
+        if str(itm_lst[-1]) == u"nan" or itm_lst[-1] is None:
+            tbl_lst_none.append(itm_lst)
+        else:
+            tbl_lst.append(itm_lst)
 
 
+    tbl_lst_none.sort(key=lambda rel: rel[0], reverse=False)
     tbl_lst.sort(key=lambda rel: rel[0], reverse=False)
     tbl_lst.sort(key=lambda rel: rel[0], reverse=False)
-    tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
+    tbl_lst.sort(key=lambda rel: rel[-1], reverse=False)
+    tbl_lst.extend(tbl_lst_none)
 
     # Generate csv table:
 
     # Generate csv table:
-    csv_file = f"{table[u'output-file']}.csv"
-    with open(csv_file, u"wt", encoding='utf-8') as file_handler:
+    csv_file_name = f"{table[u'output-file']}.csv"
+    logging.info(f"    Writing the file {csv_file_name}")
+    with open(csv_file_name, u"wt", encoding='utf-8') as file_handler:
         for hdr in header:
             file_handler.write(u",".join(hdr) + u"\n")
         for test in tbl_lst:
         for hdr in header:
             file_handler.write(u",".join(hdr) + u"\n")
         for test in tbl_lst:
@@ -1914,17 +2130,18 @@ def table_weekly_comparison(table, in_data):
                 ]
             ) + u"\n")
 
                 ]
             ) + u"\n")
 
-    txt_file = f"{table[u'output-file']}.txt"
-    convert_csv_to_pretty_txt(csv_file, txt_file, delimiter=u",")
+    txt_file_name = f"{table[u'output-file']}.txt"
+    logging.info(f"    Writing the file {txt_file_name}")
+    convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u",")
 
     # Reorganize header in txt table
     txt_table = list()
 
     # Reorganize header in txt table
     txt_table = list()
-    with open(txt_file, u"rt", encoding='utf-8') as file_handler:
-        for line in file_handler:
+    with open(txt_file_name, u"rt", encoding='utf-8') as file_handler:
+        for line in list(file_handler):
             txt_table.append(line)
     try:
         txt_table.insert(5, txt_table.pop(2))
             txt_table.append(line)
     try:
         txt_table.insert(5, txt_table.pop(2))
-        with open(txt_file, u"wt", encoding='utf-8') as file_handler:
+        with open(txt_file_name, u"wt", encoding='utf-8') as file_handler:
             file_handler.writelines(txt_table)
     except IndexError:
         pass
             file_handler.writelines(txt_table)
     except IndexError:
         pass
Integration tests