import logging
from collections import OrderedDict
-from os import remove
+from os import remove, walk, listdir
+from os.path import isfile, isdir, join
from datetime import datetime as dt
from datetime import timedelta
from json import loads
from resources.libraries.python import jumpavg
from input_data_files import download_and_unzip_data_file
+from pal_errors import PresentationError
# Separator used in file names
# Use whole list in CSIT-1180.
stats = jumpavg.AvgStdevStats.for_runs(items_float)
test_result[u"result"][u"receive-rate"] = stats.avg
+ test_result[u"result"][u"receive-stdev"] = stats.stdev
else:
groups = re.search(self.REGEX_MRR, test.message)
test_result[u"result"][u"receive-rate"] = \
:type msg: Message
:returns: Nothing.
"""
-
if self._msg_type:
self.parse_msg[self._msg_type](msg)
:returns: Metadata
:rtype: pandas.Series
"""
-
return self.data[job][build][u"metadata"]
def suites(self, job, build):
:returns: Suites.
:rtype: pandas.Series
"""
-
return self.data[job][str(build)][u"suites"]
def tests(self, job, build):
:returns: Tests.
:rtype: pandas.Series
"""
-
return self.data[job][build][u"tests"]
def _parse_tests(self, job, build, log):
logging.info(u"Done.")
+ def process_local_file(self, local_file, job=u"local", build_nr=1,
+ replace=True):
+ """Process local XML file given as a command-line parameter.
+
+ :param local_file: The file to process.
+ :param job: Job name.
+ :param build_nr: Build number.
+ :param replace: If True, the information about jobs and builds is
+ replaced by the new one, otherwise the new jobs and builds are
+ added.
+ :type local_file: str
+ :type job: str
+ :type build_nr: int
+ :type replace: bool
+ :raises: PresentationError if an error occurs.
+ """
+ if not isfile(local_file):
+ raise PresentationError(f"The file {local_file} does not exist.")
+
+ build = {
+ u"build": build_nr,
+ u"status": u"failed",
+ u"file-name": local_file
+ }
+ if replace:
+ self._cfg.builds = dict()
+ self._cfg.add_build(job, build)
+
+ logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
+ data = self._parse_tests(job, build, list())
+ if data is None:
+ raise PresentationError(
+ f"Error occurred while parsing the file {local_file}"
+ )
+
+ build_data = pd.Series({
+ u"metadata": pd.Series(
+ list(data[u"metadata"].values()),
+ index=list(data[u"metadata"].keys())
+ ),
+ u"suites": pd.Series(
+ list(data[u"suites"].values()),
+ index=list(data[u"suites"].keys())
+ ),
+ u"tests": pd.Series(
+ list(data[u"tests"].values()),
+ index=list(data[u"tests"].keys())
+ )
+ })
+
+ if self._input_data.get(job, None) is None:
+ self._input_data[job] = pd.Series()
+ self._input_data[job][str(build_nr)] = build_data
+
+ self._cfg.set_input_state(job, build_nr, u"processed")
+
+ def process_local_directory(self, local_dir, replace=True):
+ """Process local directory with XML file(s). The directory is processed
+ as a 'job' and the XML files in it as builds.
+ If the given directory contains only sub-directories, these
+ sub-directories processed as jobs and corresponding XML files as builds
+ of their job.
+
+ :param local_dir: Local directory to process.
+ :param replace: If True, the information about jobs and builds is
+ replaced by the new one, otherwise the new jobs and builds are
+ added.
+ :type local_dir: str
+ :type replace: bool
+ """
+ if not isdir(local_dir):
+ raise PresentationError(
+ f"The directory {local_dir} does not exist."
+ )
+
+ # Check if the given directory includes only files, or only directories
+ _, dirnames, filenames = next(walk(local_dir))
+
+ if filenames and not dirnames:
+ filenames.sort()
+ # local_builds:
+ # key: dir (job) name, value: list of file names (builds)
+ local_builds = {
+ local_dir: [join(local_dir, name) for name in filenames]
+ }
+
+ elif dirnames and not filenames:
+ dirnames.sort()
+ # local_builds:
+ # key: dir (job) name, value: list of file names (builds)
+ local_builds = dict()
+ for dirname in dirnames:
+ builds = [
+ join(local_dir, dirname, name)
+ for name in listdir(join(local_dir, dirname))
+ if isfile(join(local_dir, dirname, name))
+ ]
+ if builds:
+ local_builds[dirname] = sorted(builds)
+
+ elif not filenames and not dirnames:
+ raise PresentationError(f"The directory {local_dir} is empty.")
+ else:
+ raise PresentationError(
+ f"The directory {local_dir} can include only files or only "
+ f"directories, not both.\nThe directory {local_dir} includes "
+ f"file(s):\n{filenames}\nand directories:\n{dirnames}"
+ )
+
+ if replace:
+ self._cfg.builds = dict()
+
+ for job, files in local_builds.items():
+ for idx, local_file in enumerate(files):
+ self.process_local_file(local_file, job, idx + 1, replace=False)
+
@staticmethod
def _end_of_tag(tag_filter, start=0, closer=u"'"):
"""Return the index of character in the string which is the end of tag.
:returns: The index of the tag closer.
:rtype: int
"""
-
try:
idx_opener = tag_filter.index(closer, start)
return tag_filter.index(closer, idx_opener + 1)
:returns: Conditional statement which can be evaluated.
:rtype: str
"""
-
index = 0
while True:
index = InputData._end_of_tag(tag_filter, index)
"""Filter required data from the given jobs and builds.
The output data structure is:
-
- job 1
- build 1
- test (or suite) 1 ID:
"""Filter required data from the given jobs and builds.
The output data structure is:
-
- job 1
- build 1
- test (or suite) 1 ID:
for item in builds.values:
for item_id, item_data in item.items():
merged_data[item_id] = item_data
-
return merged_data
def print_all_oper_data(self):
Code Review / csit.git / blobdiff