CSIT-1078: Optimize input data files download and processing

[csit.git] / resources / tools / presentation / input_data_parser.py

diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index e12e2fb..d0f9eed 100644 (file)
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -251,6 +251,7 @@ class ExecutionChecker(ResultVisitor):
             self._version = str(re.search(self.REGEX_VERSION, msg.message).
                                 group(2))
             self._data["metadata"]["version"] = self._version
             self._version = str(re.search(self.REGEX_VERSION, msg.message).
                                 group(2))
             self._data["metadata"]["version"] = self._version

+            self._data["metadata"]["generated"] = msg.timestamp

             self._msg_type = None
 
     def _get_vat_history(self, msg):
             self._msg_type = None
 
     def _get_vat_history(self, msg):


@@ -748,17 +749,14 @@ class InputData(object):
         return self.data[job][build]["tests"]
 
     @staticmethod
         return self.data[job][build]["tests"]
 
     @staticmethod

-    def _parse_tests(job, build, get_timestamp=False):
+    def _parse_tests(job, build):

         """Process data from robot output.xml file and return JSON structured
         data.
 
         :param job: The name of job which build output data will be processed.
         :param build: The build which output data will be processed.
         """Process data from robot output.xml file and return JSON structured
         data.
 
         :param job: The name of job which build output data will be processed.
         :param build: The build which output data will be processed.

-        :param get_timestamp: If True, timestamp is read form the xml source
-            file.

         :type job: str
         :type build: dict
         :type job: str
         :type build: dict

-        :type get_timestamp: bool

         :returns: JSON data structure.
         :rtype: dict
         """
         :returns: JSON data structure.
         :rtype: dict
         """


@@ -767,10 +765,6 @@ class InputData(object):
             "job": job,
             "build": build
         }
             "job": job,
             "build": build
         }

-        if get_timestamp:
-            tree = ET.parse(build["file-name"])
-            root = tree.getroot()
-            metadata["generated"] = root.attrib["generated"]

 
         with open(build["file-name"], 'r') as data_file:
             try:
 
         with open(build["file-name"], 'r') as data_file:
             try:


@@ -784,20 +778,16 @@ class InputData(object):
 
         return checker.data
 
 
         return checker.data
 

-    def download_and_parse_data(self, get_timestamp=False):
+    def download_and_parse_data(self):

         """Download the input data files, parse input data from input files and
         store in pandas' Series.
         """Download the input data files, parse input data from input files and
         store in pandas' Series.

-
-        :param get_timestamp: If True, timestamp is read form the xml source
-            file.
-        :type get_timestamp: bool

         """
 
         logging.info("Downloading and parsing input files ...")
 
         job_data = dict()
         for job, builds in self._cfg.builds.items():
         """
 
         logging.info("Downloading and parsing input files ...")
 
         job_data = dict()
         for job, builds in self._cfg.builds.items():

-            logging.info("  Processing data from the job '{0}' ...'".
+            logging.info("  Processing data from the job '{0}' ...".

                          format(job))
             builds_data = dict()
             for build in builds:
                          format(job))
             builds_data = dict()
             for build in builds:


@@ -813,8 +803,7 @@ class InputData(object):
 
                 logging.info("      Processing data from the build '{0}' ...".
                              format(build["build"]))
 
                 logging.info("      Processing data from the build '{0}' ...".
                              format(build["build"]))

-                data = InputData._parse_tests(job, build,
-                                              get_timestamp=get_timestamp)
+                data = InputData._parse_tests(job, build)

                 if data is None:
                     logging.error("Input data file from the job '{job}', build "
                                   "'{build}' is damaged. Skipped.".
                 if data is None:
                     logging.error("Input data file from the job '{job}', build "
                                   "'{build}' is damaged. Skipped.".