CSIT-1340: Fix the list of failed tests in Trending

[csit.git] / resources / tools / presentation / generator_tables.py

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index b24222c..7590daa 100644 (file)
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -23,6 +23,8 @@ from string import replace
 from collections import OrderedDict
 from numpy import nan, isnan
 from xml.etree import ElementTree as ET
+from datetime import datetime as dt
+from datetime import timedelta
 
 from utils import mean, stdev, relative_change, classify_anomalies, \
     convert_csv_to_pretty_txt
@@ -240,11 +242,14 @@ def table_performance_comparison(table, input_data):
                     replace("1t1c", "1c").replace("2t1c", "1c").\
                     replace("2t2c", "2c").replace("4t2c", "2c").\
                     replace("4t4c", "4c").replace("8t4c", "4c")
+                if "across topologies" in table["title"].lower():
+                    tst_name_mod = tst_name_mod.replace("2n1l-", "")
                 if tbl_dict.get(tst_name_mod, None) is None:
                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
                                             "-".join(tst_data["name"].
                                                      split("-")[:-1]))
-                    if "comparison across testbeds" in table["title"].lower():
+                    if "across testbeds" in table["title"].lower() or \
+                            "across topologies" in table["title"].lower():
                         name = name.\
                             replace("1t1c", "1c").replace("2t1c", "1c").\
                             replace("2t2c", "2c").replace("4t2c", "2c").\
@@ -286,6 +291,8 @@ def table_performance_comparison(table, input_data):
                     replace("1t1c", "1c").replace("2t1c", "1c").\
                     replace("2t2c", "2c").replace("4t2c", "2c").\
                     replace("4t4c", "4c").replace("8t4c", "4c")
+                if "across topologies" in table["title"].lower():
+                    tst_name_mod = tst_name_mod.replace("2n1l-", "")
                 try:
                     # TODO: Re-work when NDRPDRDISC tests are not used
                     if table["include-tests"] == "MRR":
@@ -323,6 +330,8 @@ def table_performance_comparison(table, input_data):
                             replace("1t1c", "1c").replace("2t1c", "1c").\
                             replace("2t2c", "2c").replace("4t2c", "2c").\
                             replace("4t4c", "4c").replace("8t4c", "4c")
+                        if "across topologies" in table["title"].lower():
+                            tst_name_mod = tst_name_mod.replace("2n1l-", "")
                         if tbl_dict.get(tst_name_mod, None) is None:
                             continue
                         if tbl_dict[tst_name_mod].get("history", None) is None:
@@ -758,6 +767,10 @@ def table_failed_tests(table, input_data):
 
     # Generate the data for the table according to the model in the table
     # specification
+
+    now = dt.utcnow()
+    timeperiod = timedelta(int(table.get("window", 7)))
+
     tbl_dict = dict()
     for job, builds in table["data"].items():
         for build in builds:
@@ -774,19 +787,24 @@ def table_failed_tests(table, input_data):
                         "name": "{0}-{1}".format(nic, tst_data["name"]),
                         "data": OrderedDict()}
                 try:
-                    tbl_dict[tst_name]["data"][build] = (
-                        tst_data["status"],
-                        input_data.metadata(job, build).get("generated", ""),
-                        input_data.metadata(job, build).get("version", ""),
-                        build)
+                    generated = input_data.metadata(job, build).\
+                        get("generated", "")
+                    if not generated:
+                        continue
+                    then = dt.strptime(generated, "%Y%m%d %H:%M")
+                    if (now - then) <= timeperiod:
+                        tbl_dict[tst_name]["data"][build] = (
+                            tst_data["status"],
+                            generated,
+                            input_data.metadata(job, build).get("version", ""),
+                            build)
                 except (TypeError, KeyError):
                     pass  # No data in output.xml for this test
 
     tbl_lst = list()
     for tst_data in tbl_dict.values():
-        win_size = min(len(tst_data["data"]), table["window"])
         fails_nr = 0
-        for val in tst_data["data"].values()[-win_size:]:
+        for val in tst_data["data"].values():
             if val[0] == "FAIL":
                 fails_nr += 1
                 fails_last_date = val[1]