CSIT-1377: Implement systematic solution using defined time period

Change-Id: I03d8a5fcf9e6392e238e04a06c0463c17b5a4a47
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/resources/tools/presentation/generator_CPTA.py b/resources/tools/presentation/generator_CPTA.py
index f230a64..7d5af3f 100644 (file)
--- a/resources/tools/presentation/generator_CPTA.py
+++ b/resources/tools/presentation/generator_CPTA.py
@@ -427,7 +427,8 @@ def _generate_all_charts(spec, input_data):
             builds_dict[job] = list()
         for build in spec.input["builds"][job]:
             status = build["status"]
             builds_dict[job] = list()
         for build in spec.input["builds"][job]:
             status = build["status"]

-            if status != "failed" and status != "not found":
+            if status != "failed" and status != "not found" and \
+                status != "removed":

                 builds_dict[job].append(str(build["build"]))
 
     # Create "build ID": "date" dict:
                 builds_dict[job].append(str(build["build"]))
 
     # Create "build ID": "date" dict:

diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index 89e11bb..fe62136 100644 (file)
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -30,12 +30,19 @@ from robot import errors
 from collections import OrderedDict
 from string import replace
 from os import remove
 from collections import OrderedDict
 from string import replace
 from os import remove

+from os.path import join
+from datetime import datetime as dt
+from datetime import timedelta

 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
 
 from input_data_files import download_and_unzip_data_file
 from utils import Worker
 
 
 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
 
 from input_data_files import download_and_unzip_data_file
 from utils import Worker
 
 

+# Separator used in file names
+SEPARATOR = "__"
+
+

 class ExecutionChecker(ResultVisitor):
     """Class to traverse through the test suite structure.
 
 class ExecutionChecker(ResultVisitor):
     """Class to traverse through the test suite structure.
 


@@ -696,7 +703,7 @@ class ExecutionChecker(ResultVisitor):
                 else:
                     test_result["status"] = "FAIL"
                     self._data["tests"][self._test_ID] = test_result
                 else:
                     test_result["status"] = "FAIL"
                     self._data["tests"][self._test_ID] = test_result

-                    logging.error("The test '{0}' has no or more than one "
+                    logging.debug("The test '{0}' has no or more than one "

                                   "multi-threading tags.".format(self._test_ID))
                     logging.debug("Tags: {0}".format(test_result["tags"]))
                     return
                                   "multi-threading tags.".format(self._test_ID))
                     logging.debug("Tags: {0}".format(test_result["tags"]))
                     return


@@ -1143,7 +1150,45 @@ class InputData(object):
                 remove(build["file-name"])
             except OSError as err:
                 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
                 remove(build["file-name"])
             except OSError as err:
                 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".

-                             format(build["file-name"], err)))
+                             format(build["file-name"], repr(err))))
+
+        # If the time-period is defined in the specification file, remove all
+        # files which are outside the time period.
+        timeperiod = self._cfg.input.get("time-period", None)
+        if timeperiod and data:
+            now = dt.utcnow()
+            timeperiod = timedelta(int(timeperiod))
+            metadata = data.get("metadata", None)
+            if metadata:
+                generated = metadata.get("generated", None)
+                if generated:
+                    generated = dt.strptime(generated, "%Y%m%d %H:%M")
+                    if (now - generated) > timeperiod:
+                        # Remove the data and the file:
+                        state = "removed"
+                        data = None
+                        logs.append(
+                            ("INFO",
+                             "    The build {job}/{build} is outdated, will be "
+                             "removed".format(job=job, build=build["build"])))
+                        file_name = self._cfg.input["file-name"]
+                        full_name = join(
+                            self._cfg.environment["paths"]["DIR[WORKING,DATA]"],
+                            "{job}{sep}{build}{sep}{name}".
+                                format(job=job,
+                                       sep=SEPARATOR,
+                                       build=build["build"],
+                                       name=file_name))
+                        try:
+                            remove(full_name)
+                            logs.append(("INFO",
+                                         "    The file {name} has been removed".
+                                         format(name=full_name)))
+                        except OSError as err:
+                            logs.append(("ERROR",
+                                        "Cannot remove the file '{0}': {1}".
+                                        format(full_name, repr(err))))
+

         logs.append(("INFO", "  Done."))
 
         result = {
         logs.append(("INFO", "  Done."))
 
         result = {

diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index 760f087..1314ef4 100644 (file)
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -274,6 +274,9 @@
     file-format: ".gz"
     download-path: "{job}/{build}/archives/{filename}"
     extract: "output.xml"
     file-format: ".gz"
     download-path: "{job}/{build}/archives/{filename}"
     extract: "output.xml"

+    # Number of days from now to the past. Only files generated in this
+    # time period are used.
+    time-period: 90

 
   builds:
 
 
   builds:
 

diff --git a/resources/tools/presentation/utils.py b/resources/tools/presentation/utils.py
index 51bb1d0..3fdec85 100644 (file)
--- a/resources/tools/presentation/utils.py
+++ b/resources/tools/presentation/utils.py
@@ -25,6 +25,7 @@ from os import walk, makedirs, environ
 from os.path import join, isdir
 from shutil import move, Error
 from math import sqrt
 from os.path import join, isdir
 from shutil import move, Error
 from math import sqrt

+from datetime import datetime

 
 from errors import PresentationError
 from jumpavg.BitCountingClassifier import BitCountingClassifier
 
 from errors import PresentationError
 from jumpavg.BitCountingClassifier import BitCountingClassifier


@@ -178,6 +179,29 @@ def get_last_completed_build_number(jenkins_url, job_name):
     return execute_command(cmd)
 
 
     return execute_command(cmd)
 
 

+def get_build_timestamp(jenkins_url, job_name, build_nr):
+    """Get the timestamp of the build of the given job.
+
+    :param jenkins_url: Jenkins URL.
+    :param job_name: Job name.
+    :param build_nr: Build number.
+    :type jenkins_url: str
+    :type job_name: str
+    :type build_nr: int
+    :returns: The timestamp.
+    :rtype: datetime.datetime
+    """
+
+    url = "{jenkins_url}/{job_name}/{build_nr}".format(jenkins_url=jenkins_url,
+                                                       job_name=job_name,
+                                                       build_nr=build_nr)
+    cmd = "wget -qO- {url}".format(url=url)
+
+    timestamp = execute_command(cmd)
+
+    return datetime.fromtimestamp(timestamp/1000)
+
+

 def archive_input_data(spec):
     """Archive the report.
 
 def archive_input_data(spec):
     """Archive the report.