Trending: Reduce input data

Change-Id: Ia787036e4a717b9693df09fc75738369d6d39b57
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/resources/tools/presentation/generator_alerts.py b/resources/tools/presentation/generator_alerts.py
index ce5d803..b106d95 100644 (file)
--- a/resources/tools/presentation/generator_alerts.py
+++ b/resources/tools/presentation/generator_alerts.py
@@ -142,19 +142,8 @@ class Alerting(object):
         """
 
         for alert, alert_data in self.alerts.iteritems():
-            if alert_data["way"] == "email":
-                text, html = self._create_alert_message(alert_data)
-                conf = self.configs["email"]
-                self._send_email(server=conf["server"],
-                                 addr_from=conf["address-from"],
-                                 addr_to=conf["address-to"],
-                                 subject=alert_data["title"],
-                                 text=text,
-                                 html=html)
-            elif alert_data["way"] == "jenkins":
+            if alert_data["way"] == "jenkins":
                 self._generate_email_body(alert_data)
-                # TODO: Remove when not needed
-                self._generate_files_for_jenkins(alert_data)
             else:
                 raise AlertingError("Alert with way '{0}' is not implemented.".
                                     format(alert_data["way"]))
@@ -253,9 +242,10 @@ class Alerting(object):
 
         directory = self.configs[alert["way"]]["output-dir"]
         failed_tests = OrderedDict()
+        file_path = "{0}/{1}.txt".format(directory, test_set)
         version = ""
         try:
-            with open("{0}/{1}.txt".format(directory, test_set), 'r') as f_txt:
+            with open(file_path, 'r') as f_txt:
                 for idx, line in enumerate(f_txt):
                     if idx == 0:
                         build = line[:-1]
@@ -281,8 +271,9 @@ class Alerting(object):
                         failed_tests[name]["framesizes"].append(framesize)
                     if cores not in failed_tests[name]["cores"]:
                         failed_tests[name]["cores"].append(cores)
-        except IOError as err:
-            logging.error(repr(err))
+        except IOError:
+            logging.error("No such file or directory: {file}".
+                          format(file=file_path))
             return None, None, None, None
         if sort:
             sorted_failed_tests = OrderedDict()
@@ -312,6 +303,8 @@ class Alerting(object):
             build, version, nr, failed_tests = \
                 self._get_compressed_failed_tests(alert, test_set)
             if build is None:
+                text += "\n\nNo data for the test set '{set}'.\n".\
+                    format(set=test_set)
                 continue
             text += ("\n\n{topo}-{arch}, "
                      "{nr} tests failed, "
@@ -363,24 +356,3 @@ class Alerting(object):
         except IOError:
             logging.error("Not possible to write the file '{0}.txt'.".
                           format(file_name))
-
-    def _generate_files_for_jenkins(self, alert):
-        """Create the file which is used in the generated alert.
-
-        # TODO: Remove when not needed.
-
-        :param alert: Files are created for this alert.
-        :type alert: dict
-        """
-
-        config = self.configs[alert["way"]]
-
-        zip_file = config.get("zip-output", None)
-        if zip_file:
-            logging.info("Writing the file '{0}/{1}' ...".
-                         format(config["output-dir"], zip_file))
-            execute_command("tar czvf {dir}/{zip} --directory={dir} "
-                            "{input}.txt".
-                            format(dir=config["output-dir"],
-                                   zip=zip_file,
-                                   input=config["output-file"]))

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index cdce5f9..6f9925f 100644 (file)
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -885,8 +885,14 @@ def table_last_failed_tests(table, input_data):
     for job, builds in table["data"].items():
         for build in builds:
             build = str(build)
+            try:
+                version = input_data.metadata(job, build).get("version", "")
+            except KeyError:
+                logging.error("Data for {job}: {build} is not present.".
+                              format(job=job, build=build))
+                return
             tbl_list.append(build)
-            tbl_list.append(input_data.metadata(job, build).get("version", ""))
+            tbl_list.append(version)
             for tst_name, tst_data in data[job][build].iteritems():
                 if tst_data["status"] != "FAIL":
                     continue

diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index 9604da5..cc6bb01 100644 (file)
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -169,7 +169,7 @@
     # 3n-hsw
     plot-performance-trending-all-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 487
+        start: 501
         end: "lastCompletedBuild"
         skip:
           - 593
@@ -185,7 +185,7 @@
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 487
+        start: 501
         end: "lastCompletedBuild"
         skip:
           - 593
@@ -204,7 +204,7 @@
     # 3n-skx
     plot-performance-trending-all-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 237
+        start: 250
         end: "lastCompletedBuild"
         skip:
           - 356
@@ -237,7 +237,7 @@
 
     plot-performance-trending-vpp-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 237
+        start: 250
         end: "lastCompletedBuild"
         skip:
           - 356
@@ -273,7 +273,7 @@
     # 2n-skx
     plot-performance-trending-all-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 237
+        start: 250
         end: "lastCompletedBuild"
         skip:
           - 347
@@ -291,13 +291,14 @@
           - 375
           - 380
           - 381
+          - 396
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
         start: 20
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 237
+        start: 250
         end: "lastCompletedBuild"
         skip:
           - 347
@@ -315,6 +316,7 @@
           - 375
           - 380
           - 381
+          - 396
 
     plot-performance-trending-dpdk-2n-skx:
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
@@ -419,7 +421,7 @@
 
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:
-      start: 487
+      start: 501
       end: "lastCompletedBuild"
       skip:
         - 593
@@ -435,7 +437,7 @@
 
     # 3n-skx
     csit-vpp-perf-mrr-daily-master-3n-skx:
-      start: 237
+      start: 250
       end: "lastCompletedBuild"
       skip:
         - 356
@@ -468,7 +470,7 @@
 
     # 2n-skx
     csit-vpp-perf-mrr-daily-master-2n-skx:
-      start: 237
+      start: 250
       end: "lastCompletedBuild"
       skip:
         - 347
@@ -486,6 +488,7 @@
         - 375
         - 380
         - 381
+        - 396
     csit-vpp-perf-mrr-weekly-master-2n-skx:
       start: 1
       end: "lastCompletedBuild"