X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=8b88a52a79c14d8f83aa92cec0f521aeb21df3ff;hb=ece79f53197aa36aa356f7fa1ce1dfe0b582ae86;hp=42f16e4b338b585c9837cba870971dc15431a082;hpb=3c88629c05256b8f5b1e435a8e94d488eb769224;p=csit.git

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 42f16e4b33..8b88a52a79 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -17,6 +17,7 @@
 
 import logging
 import csv
+import math
 import re
 
 from collections import OrderedDict
@@ -24,7 +25,6 @@ from xml.etree import ElementTree as ET
 from datetime import datetime as dt
 from datetime import timedelta
 from copy import deepcopy
-from json import loads
 
 import plotly.graph_objects as go
 import plotly.offline as ploff
@@ -477,6 +477,13 @@ def _tpc_insert_data(target, src, include_tests):
             target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
         elif include_tests == u"NDR":
             target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
+        elif u"latency" in include_tests:
+            keys = include_tests.split(u"-")
+            if len(keys) == 4:
+                lat = src[keys[0]][keys[1]][keys[2]][keys[3]]
+                target[u"data"].append(
+                    float(u"nan") if lat == -1 else lat * 1e6
+                )
     except (KeyError, TypeError):
         pass
 
@@ -917,7 +924,11 @@ def table_perf_trending_dash(table, input_data):
         if len(data_t) < 2:
             continue
 
-        classification_lst, avgs, _ = classify_anomalies(data_t)
+        try:
+            classification_lst, avgs, _ = classify_anomalies(data_t)
+        except ValueError as err:
+            logging.info(f"{err} Skipping")
+            return
 
         win_size = min(len(data_t), table[u"window"])
         long_win_size = min(len(data_t), table[u"long-trend-window"])
@@ -1230,6 +1241,9 @@ def table_perf_trending_dash_html(table, input_data):
     try:
         with open(table[u"input-file"], u'rt') as csv_file:
             csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
+    except FileNotFoundError as err:
+        logging.warning(f"{err}")
+        return
     except KeyError:
         logging.warning(u"The input file is not defined.")
         return
@@ -1341,6 +1355,8 @@ def table_last_failed_tests(table, input_data):
             build = str(build)
             try:
                 version = input_data.metadata(job, build).get(u"version", u"")
+                duration = \
+                    input_data.metadata(job, build).get(u"elapsedtime", u"")
             except KeyError:
                 logging.error(f"Data for {job}: {build} is not present.")
                 return
@@ -1358,16 +1374,21 @@ def table_last_failed_tests(table, input_data):
                 if not groups:
                     continue
                 nic = groups.group(0)
-                failed_tests.append(f"{nic}-{tst_data[u'name']}")
-            tbl_list.append(str(passed))
-            tbl_list.append(str(failed))
+                msg = tst_data[u'msg'].replace(u"\n", u"")
+                msg = re.sub(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})',
+                             'xxx.xxx.xxx.xxx', msg)
+                msg = msg.split(u'Also teardown failed')[0]
+                failed_tests.append(f"{nic}-{tst_data[u'name']}###{msg}")
+            tbl_list.append(passed)
+            tbl_list.append(failed)
+            tbl_list.append(duration)
             tbl_list.extend(failed_tests)
 
     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
     logging.info(f"    Writing file: {file_name}")
     with open(file_name, u"wt") as file_handler:
         for test in tbl_list:
-            file_handler.write(test + u'\n')
+            file_handler.write(f"{test}\n")
 
 
 def table_failed_tests(table, input_data):
@@ -1613,7 +1634,14 @@ def table_comparison(table, input_data):
         tag = col.get(u"tag", None)
         data = input_data.filter_data(
             table,
-            params=[u"throughput", u"result", u"name", u"parent", u"tags"],
+            params=[
+                u"throughput",
+                u"result",
+                u"latency",
+                u"name",
+                u"parent",
+                u"tags"
+            ],
             data=col[u"data-set"],
             continue_on_error=True
         )
@@ -1651,7 +1679,14 @@ def table_comparison(table, input_data):
         if replacement:
             rpl_data = input_data.filter_data(
                 table,
-                params=[u"throughput", u"result", u"name", u"parent", u"tags"],
+                params=[
+                    u"throughput",
+                    u"result",
+                    u"latency",
+                    u"name",
+                    u"parent",
+                    u"tags"
+                ],
                 data=replacement,
                 continue_on_error=True
             )
@@ -1685,7 +1720,8 @@ def table_comparison(table, input_data):
                             include_tests=table[u"include-tests"]
                         )
 
-        if table[u"include-tests"] in (u"NDR", u"PDR"):
+        if table[u"include-tests"] in (u"NDR", u"PDR") or \
+                u"latency" in table[u"include-tests"]:
             for tst_name, tst_data in col_data[u"data"].items():
                 if tst_data[u"data"]:
                     tst_data[u"mean"] = mean(tst_data[u"data"])
@@ -1770,11 +1806,14 @@ def table_comparison(table, input_data):
                         cmp_itm[u"mean"] is not None and \
                         ref_itm[u"stdev"] is not None and \
                         cmp_itm[u"stdev"] is not None:
-                    delta, d_stdev = relative_change_stdev(
-                        ref_itm[u"mean"], cmp_itm[u"mean"],
-                        ref_itm[u"stdev"], cmp_itm[u"stdev"]
-                    )
-                    if delta is None:
+                    try:
+                        delta, d_stdev = relative_change_stdev(
+                            ref_itm[u"mean"], cmp_itm[u"mean"],
+                            ref_itm[u"stdev"], cmp_itm[u"stdev"]
+                        )
+                    except ZeroDivisionError:
+                        break
+                    if delta is None or math.isnan(delta):
                         break
                     new_row.append({
                         u"mean": delta * 1e6,