Trending: Align the content of old and new dirs

Change-Id: I39d1f71adac0bad3dbe18a987909904ea5c1e6cb
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/resources/tools/presentation/generator_alerts.py b/resources/tools/presentation/generator_alerts.py
index cbdb1fd..83dfe2e 100644 (file)
--- a/resources/tools/presentation/generator_alerts.py
+++ b/resources/tools/presentation/generator_alerts.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -70,7 +70,13 @@ class Alerting(object):
         # Implemented alerts:
         self._ALERTS = ("failed-tests", )
 
         # Implemented alerts:
         self._ALERTS = ("failed-tests", )
 

-        self._spec = spec.alerting
+        try:
+            self._spec = spec.alerting
+        except KeyError as err:
+            raise  AlertingError("Alerting is not configured, skipped.",
+                                 repr(err),
+                                 "WARNING")
+

         self._path_failed_tests = spec.environment["paths"]["DIR[STATIC,VPP]"]
 
         # Verify and validate input specification:
         self._path_failed_tests = spec.environment["paths"]["DIR[STATIC,VPP]"]
 
         # Verify and validate input specification:

diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py
index 5e41921..39cba09 100644 (file)
--- a/resources/tools/presentation/pal.py
+++ b/resources/tools/presentation/pal.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -112,8 +112,11 @@ def main():
             logging.info("Successfully finished.")
         elif spec.output["output"] == "CPTA":
             sys.stdout.write(generate_cpta(spec, data))
             logging.info("Successfully finished.")
         elif spec.output["output"] == "CPTA":
             sys.stdout.write(generate_cpta(spec, data))

-            alert = Alerting(spec)
-            alert.generate_alerts()
+            try:
+                alert = Alerting(spec)
+                alert.generate_alerts()
+            except AlertingError as err:
+                logging.warning(repr(err))

             logging.info("Successfully finished.")
         ret_code = 0
 
             logging.info("Successfully finished.")
         ret_code = 0
 

diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index 846fa71..45ea85c 100644 (file)
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -140,20 +140,20 @@
     # 3n-hsw
     plot-performance-trending-all-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
     # 3n-hsw
     plot-performance-trending-all-3n-hsw:
       csit-vpp-perf-mrr-daily-master:

-        start: 290
+        start: 300

         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master:
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master:

-        start: 38
+        start: 50

         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:

-        start: 290
+        start: 300

         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-hsw:
       csit-dpdk-perf-mrr-weekly-master:
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-hsw:
       csit-dpdk-perf-mrr-weekly-master:

-        start: 38
+        start: 50

         end: "lastCompletedBuild"
 
     # 3n-skx
         end: "lastCompletedBuild"
 
     # 3n-skx


@@ -282,10 +282,10 @@
 
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:
 
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:

-      start: 290
+      start: 300

       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master:
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master:

-      start: 38
+      start: 50

       end: "lastCompletedBuild"
 
     # 3n-skx
       end: "lastCompletedBuild"
 
     # 3n-skx

diff --git a/resources/tools/presentation_new/generator_CPTA.py b/resources/tools/presentation_new/generator_CPTA.py
index f230a64..24dc28c 100644 (file)
--- a/resources/tools/presentation_new/generator_CPTA.py
+++ b/resources/tools/presentation_new/generator_CPTA.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -427,7 +427,8 @@ def _generate_all_charts(spec, input_data):
             builds_dict[job] = list()
         for build in spec.input["builds"][job]:
             status = build["status"]
             builds_dict[job] = list()
         for build in spec.input["builds"][job]:
             status = build["status"]

-            if status != "failed" and status != "not found":
+            if status != "failed" and status != "not found" and \
+                status != "removed":

                 builds_dict[job].append(str(build["build"]))
 
     # Create "build ID": "date" dict:
                 builds_dict[job].append(str(build["build"]))
 
     # Create "build ID": "date" dict:

diff --git a/resources/tools/presentation_new/generator_alerts.py b/resources/tools/presentation_new/generator_alerts.py
index cbdb1fd..83dfe2e 100644 (file)
--- a/resources/tools/presentation_new/generator_alerts.py
+++ b/resources/tools/presentation_new/generator_alerts.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -70,7 +70,13 @@ class Alerting(object):
         # Implemented alerts:
         self._ALERTS = ("failed-tests", )
 
         # Implemented alerts:
         self._ALERTS = ("failed-tests", )
 

-        self._spec = spec.alerting
+        try:
+            self._spec = spec.alerting
+        except KeyError as err:
+            raise  AlertingError("Alerting is not configured, skipped.",
+                                 repr(err),
+                                 "WARNING")
+

         self._path_failed_tests = spec.environment["paths"]["DIR[STATIC,VPP]"]
 
         # Verify and validate input specification:
         self._path_failed_tests = spec.environment["paths"]["DIR[STATIC,VPP]"]
 
         # Verify and validate input specification:

diff --git a/resources/tools/presentation_new/input_data_parser.py b/resources/tools/presentation_new/input_data_parser.py
index 0712156..f7a2094 100644 (file)
--- a/resources/tools/presentation_new/input_data_parser.py
+++ b/resources/tools/presentation_new/input_data_parser.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -30,12 +30,19 @@ from robot import errors
 from collections import OrderedDict
 from string import replace
 from os import remove
 from collections import OrderedDict
 from string import replace
 from os import remove

+from os.path import join
+from datetime import datetime as dt
+from datetime import timedelta

 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
 
 from input_data_files import download_and_unzip_data_file
 from utils import Worker
 
 
 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
 
 from input_data_files import download_and_unzip_data_file
 from utils import Worker
 
 

+# Separator used in file names
+SEPARATOR = "__"
+
+

 class ExecutionChecker(ResultVisitor):
     """Class to traverse through the test suite structure.
 
 class ExecutionChecker(ResultVisitor):
     """Class to traverse through the test suite structure.
 


@@ -696,7 +703,7 @@ class ExecutionChecker(ResultVisitor):
                 else:
                     test_result["status"] = "FAIL"
                     self._data["tests"][self._test_ID] = test_result
                 else:
                     test_result["status"] = "FAIL"
                     self._data["tests"][self._test_ID] = test_result

-                    logging.error("The test '{0}' has no or more than one "
+                    logging.debug("The test '{0}' has no or more than one "

                                   "multi-threading tags.".format(self._test_ID))
                     logging.debug("Tags: {0}".format(test_result["tags"]))
                     return
                                   "multi-threading tags.".format(self._test_ID))
                     logging.debug("Tags: {0}".format(test_result["tags"]))
                     return


@@ -1139,7 +1146,45 @@ class InputData(object):
                 remove(build["file-name"])
             except OSError as err:
                 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
                 remove(build["file-name"])
             except OSError as err:
                 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".

-                             format(build["file-name"], err)))
+                             format(build["file-name"], repr(err))))
+
+        # If the time-period is defined in the specification file, remove all
+        # files which are outside the time period.
+        timeperiod = self._cfg.input.get("time-period", None)
+        if timeperiod and data:
+            now = dt.utcnow()
+            timeperiod = timedelta(int(timeperiod))
+            metadata = data.get("metadata", None)
+            if metadata:
+                generated = metadata.get("generated", None)
+                if generated:
+                    generated = dt.strptime(generated, "%Y%m%d %H:%M")
+                    if (now - generated) > timeperiod:
+                        # Remove the data and the file:
+                        state = "removed"
+                        data = None
+                        logs.append(
+                            ("INFO",
+                             "    The build {job}/{build} is outdated, will be "
+                             "removed".format(job=job, build=build["build"])))
+                        file_name = self._cfg.input["file-name"]
+                        full_name = join(
+                            self._cfg.environment["paths"]["DIR[WORKING,DATA]"],
+                            "{job}{sep}{build}{sep}{name}".
+                                format(job=job,
+                                       sep=SEPARATOR,
+                                       build=build["build"],
+                                       name=file_name))
+                        try:
+                            remove(full_name)
+                            logs.append(("INFO",
+                                         "    The file {name} has been removed".
+                                         format(name=full_name)))
+                        except OSError as err:
+                            logs.append(("ERROR",
+                                        "Cannot remove the file '{0}': {1}".
+                                        format(full_name, repr(err))))
+

         logs.append(("INFO", "  Done."))
 
         result = {
         logs.append(("INFO", "  Done."))
 
         result = {

diff --git a/resources/tools/presentation_new/pal.py b/resources/tools/presentation_new/pal.py
index 5e41921..39cba09 100644 (file)
--- a/resources/tools/presentation_new/pal.py
+++ b/resources/tools/presentation_new/pal.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -112,8 +112,11 @@ def main():
             logging.info("Successfully finished.")
         elif spec.output["output"] == "CPTA":
             sys.stdout.write(generate_cpta(spec, data))
             logging.info("Successfully finished.")
         elif spec.output["output"] == "CPTA":
             sys.stdout.write(generate_cpta(spec, data))

-            alert = Alerting(spec)
-            alert.generate_alerts()
+            try:
+                alert = Alerting(spec)
+                alert.generate_alerts()
+            except AlertingError as err:
+                logging.warning(repr(err))

             logging.info("Successfully finished.")
         ret_code = 0
 
             logging.info("Successfully finished.")
         ret_code = 0
 

diff --git a/resources/tools/presentation_new/specification_CPTA.yaml b/resources/tools/presentation_new/specification_CPTA.yaml
index 88a68d1..730f2fe 100644 (file)
--- a/resources/tools/presentation_new/specification_CPTA.yaml
+++ b/resources/tools/presentation_new/specification_CPTA.yaml
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -73,58 +73,58 @@
     # 3n-hsw
     plot-performance-trending-all-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
     # 3n-hsw
     plot-performance-trending-all-3n-hsw:
       csit-vpp-perf-mrr-daily-master:

-        start: 260
+        start: 300

         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master:
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master:

-        start: 3
+        start: 50

         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:

-        start: 260
+        start: 300

         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-hsw:
       csit-dpdk-perf-mrr-weekly-master:
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-hsw:
       csit-dpdk-perf-mrr-weekly-master:

-        start: 3
+        start: 50

         end: "lastCompletedBuild"
 
     # 3n-skx
     plot-performance-trending-all-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
         end: "lastCompletedBuild"
 
     # 3n-skx
     plot-performance-trending-all-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:

-        start: 20
+        start: 50

         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master-3n-skx:

-        start: 1
+        start: 3

         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:

-        start: 20
+        start: 50

         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-skx:
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-skx:
       csit-dpdk-perf-mrr-weekly-master-3n-skx:

-        start: 1
+        start: 3

         end: "lastCompletedBuild"
 
     # 2n-skx
     plot-performance-trending-all-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
         end: "lastCompletedBuild"
 
     # 2n-skx
     plot-performance-trending-all-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:

-        start: 20
+        start: 50

         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master-2n-skx:

-        start: 1
+        start: 3

         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:

-        start: 20
+        start: 50

         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-2n-skx:
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-2n-skx:
       csit-dpdk-perf-mrr-weekly-master-2n-skx:

-        start: 1
+        start: 3

         end: "lastCompletedBuild"
 
   plot-layouts:
         end: "lastCompletedBuild"
 
   plot-layouts:


@@ -207,31 +207,34 @@
     file-format: ".gz"
     download-path: "{job}/{build}/archives/{filename}"
     extract: "output.xml"
     file-format: ".gz"
     download-path: "{job}/{build}/archives/{filename}"
     extract: "output.xml"

+    # Number of days from now to the past. Only files generated in this
+    # time period are used.
+    time-period: 90

 
   builds:
 
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:
 
   builds:
 
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:

-      start: 260
+      start: 300

       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master:
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master:

-      start: 3
+      start: 50

       end: "lastCompletedBuild"
 
     # 3n-skx
     csit-vpp-perf-mrr-daily-master-3n-skx:
       end: "lastCompletedBuild"
 
     # 3n-skx
     csit-vpp-perf-mrr-daily-master-3n-skx:

-      start: 20
+      start: 50

       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master-3n-skx:
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master-3n-skx:

-      start: 1
+      start: 3

       end: "lastCompletedBuild"
 
     # 2n-skx
     csit-vpp-perf-mrr-daily-master-2n-skx:
       end: "lastCompletedBuild"
 
     # 2n-skx
     csit-vpp-perf-mrr-daily-master-2n-skx:

-      start: 20
+      start: 50

       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master-2n-skx:
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master-2n-skx:

-      start: 1
+      start: 3

       end: "lastCompletedBuild"
 
 -
       end: "lastCompletedBuild"
 
 -

diff --git a/resources/tools/presentation_new/utils.py b/resources/tools/presentation_new/utils.py
index 51bb1d0..3fdec85 100644 (file)
--- a/resources/tools/presentation_new/utils.py
+++ b/resources/tools/presentation_new/utils.py
@@ -25,6 +25,7 @@ from os import walk, makedirs, environ
 from os.path import join, isdir
 from shutil import move, Error
 from math import sqrt
 from os.path import join, isdir
 from shutil import move, Error
 from math import sqrt

+from datetime import datetime

 
 from errors import PresentationError
 from jumpavg.BitCountingClassifier import BitCountingClassifier
 
 from errors import PresentationError
 from jumpavg.BitCountingClassifier import BitCountingClassifier


@@ -178,6 +179,29 @@ def get_last_completed_build_number(jenkins_url, job_name):
     return execute_command(cmd)
 
 
     return execute_command(cmd)
 
 

+def get_build_timestamp(jenkins_url, job_name, build_nr):
+    """Get the timestamp of the build of the given job.
+
+    :param jenkins_url: Jenkins URL.
+    :param job_name: Job name.
+    :param build_nr: Build number.
+    :type jenkins_url: str
+    :type job_name: str
+    :type build_nr: int
+    :returns: The timestamp.
+    :rtype: datetime.datetime
+    """
+
+    url = "{jenkins_url}/{job_name}/{build_nr}".format(jenkins_url=jenkins_url,
+                                                       job_name=job_name,
+                                                       build_nr=build_nr)
+    cmd = "wget -qO- {url}".format(url=url)
+
+    timestamp = execute_command(cmd)
+
+    return datetime.fromtimestamp(timestamp/1000)
+
+

 def archive_input_data(spec):
     """Archive the report.
 
 def archive_input_data(spec):
     """Archive the report.