- extract data from output.xml files generated by Jenkins jobs and store in
pandas' Series,
- provide access to the data.
+- filter the data using tags,
"""
import multiprocessing
from collections import OrderedDict
from string import replace
from os import remove
+from os.path import join
+from datetime import datetime as dt
+from datetime import timedelta
+from json import loads
+from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
from input_data_files import download_and_unzip_data_file
from utils import Worker
+# Separator used in file names
+SEPARATOR = "__"
+
+
class ExecutionChecker(ResultVisitor):
"""Class to traverse through the test suite structure.
Performance tests:
{
- "metadata": { # Optional
- "version": "VPP version",
+ "metadata": {
+ "generated": "Timestamp",
+ "version": "SUT version",
"job": "Jenkins job name",
"build": "Information about the build"
},
"suites": {
- "Suite name 1": {
+ "Suite long name 1": {
+ "name": Suite name,
"doc": "Suite 1 documentation",
"parent": "Suite 1 parent",
"level": "Level of the suite in the suite hierarchy"
}
- "Suite name N": {
+ "Suite long name N": {
+ "name": Suite name,
"doc": "Suite N documentation",
"parent": "Suite 2 parent",
"level": "Level of the suite in the suite hierarchy"
}
}
"tests": {
+ # NDRPDR tests:
"ID": {
"name": "Test name",
"parent": "Name of the parent of the test",
- "doc": "Test documentation"
- "msg": "Test message"
+ "doc": "Test documentation",
+ "msg": "Test message",
+ "vat-history": "DUT1 and DUT2 VAT History",
+ "show-run": "Show Run",
"tags": ["tag 1", "tag 2", "tag n"],
- "type": "PDR" | "NDR",
+ "type": "NDRPDR",
+ "status": "PASS" | "FAIL",
"throughput": {
+ "NDR": {
+ "LOWER": float,
+ "UPPER": float
+ },
+ "PDR": {
+ "LOWER": float,
+ "UPPER": float
+ }
+ },
+ "latency": {
+ "NDR": {
+ "direction1": {
+ "min": float,
+ "avg": float,
+ "max": float
+ },
+ "direction2": {
+ "min": float,
+ "avg": float,
+ "max": float
+ }
+ },
+ "PDR": {
+ "direction1": {
+ "min": float,
+ "avg": float,
+ "max": float
+ },
+ "direction2": {
+ "min": float,
+ "avg": float,
+ "max": float
+ }
+ }
+ }
+ }
+
+ # TCP tests:
+ "ID": {
+ "name": "Test name",
+ "parent": "Name of the parent of the test",
+ "doc": "Test documentation",
+ "msg": "Test message",
+ "tags": ["tag 1", "tag 2", "tag n"],
+ "type": "TCP",
+ "status": "PASS" | "FAIL",
+ "result": int
+ }
+
+ # MRR, BMRR tests:
+ "ID": {
+ "name": "Test name",
+ "parent": "Name of the parent of the test",
+ "doc": "Test documentation",
+ "msg": "Test message",
+ "tags": ["tag 1", "tag 2", "tag n"],
+ "type": "MRR" | "BMRR",
+ "status": "PASS" | "FAIL",
+ "result": {
+ "receive-rate": AvgStdevMetadata,
+ }
+ }
+
+ # TODO: Remove when definitely no NDRPDRDISC tests are used:
+ # NDRPDRDISC tests:
+ "ID": {
+ "name": "Test name",
+ "parent": "Name of the parent of the test",
+ "doc": "Test documentation",
+ "msg": "Test message",
+ "tags": ["tag 1", "tag 2", "tag n"],
+ "type": "PDR" | "NDR",
+ "status": "PASS" | "FAIL",
+ "throughput": { # Only type: "PDR" | "NDR"
"value": int,
"unit": "pps" | "bps" | "percentage"
},
- "latency": {
+ "latency": { # Only type: "PDR" | "NDR"
"direction1": {
"100": {
"min": int,
}
}
},
- "lossTolerance": "lossTolerance", # Only for PDR
+ "lossTolerance": "lossTolerance", # Only type: "PDR"
"vat-history": "DUT1 and DUT2 VAT History"
- },
"show-run": "Show Run"
},
"ID" {
}
}
- Functional tests:
+ Functional tests:
{
"metadata": { # Optional
.. note:: ID is the lowercase full path to the test.
"""
+ # TODO: Remove when definitely no NDRPDRDISC tests are used:
REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
+ REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
+ r'NDR_UPPER:\s(\d+.\d+).*\n'
+ r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
+ r'PDR_UPPER:\s(\d+.\d+)')
+
+ # TODO: Remove when definitely no NDRPDRDISC tests are used:
REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
- r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+/-?\d+)\','
+ r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
+ REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
+ r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
+
REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
r'[\D\d]*')
- REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*)(.*)")
+ REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|"
+ r"VPP Version:\s*)(.*)")
REGEX_VERSION_DPDK = re.compile(r"(return STDOUT testpmd)([\d\D\n]*)"
r"(RTE Version: 'DPDK )(.*)(')")
REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
r'tx\s(\d*),\srx\s(\d*)')
- def __init__(self, metadata):
+ REGEX_BMRR = re.compile(r'Maximum Receive Rate trial results'
+ r' in packets per second: \[(.*)\]')
+
+ REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
+
+ REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
+
+ REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
+
+ REGEX_TC_NUMBER = re.compile(r'tc[0-9]{2}-')
+
+ def __init__(self, metadata, mapping, ignore):
"""Initialisation.
:param metadata: Key-value pairs to be included in "metadata" part of
- JSON structure.
+ JSON structure.
+ :param mapping: Mapping of the old names of test cases to the new
+ (actual) one.
+ :param ignore: List of TCs to be ignored.
:type metadata: dict
+ :type mapping: dict
+ :type ignore: list
"""
# Type of message to parse out from the test messages
# Timestamp
self._timestamp = None
+ # Testbed. The testbed is identified by TG node IP address.
+ self._testbed = None
+
+ # Mapping of TCs long names
+ self._mapping = mapping
+
+ # Ignore list
+ self._ignore = ignore
+
# Number of VAT History messages found:
# 0 - no message
# 1 - VAT History of DUT1
"vpp-version": self._get_vpp_version,
"dpdk-version": self._get_dpdk_version,
"teardown-vat-history": self._get_vat_history,
- "test-show-runtime": self._get_show_run
+ "test-show-runtime": self._get_show_run,
+ "testbed": self._get_testbed
}
@property
"""
return self._data
+ def _get_testbed(self, msg):
+ """Called when extraction of testbed IP is required.
+ The testbed is identified by TG node IP address.
+
+ :param msg: Message to process.
+ :type msg: Message
+ :returns: Nothing.
+ """
+
+ if msg.message.count("Arguments:"):
+ message = str(msg.message).replace(' ', '').replace('\n', '').\
+ replace("'", '"').replace('b"', '"').\
+ replace("honeycom", "honeycomb")
+ message = loads(message[11:-1])
+ try:
+ self._testbed = message["TG"]["host"]
+ except (KeyError, ValueError):
+ pass
+ finally:
+ self._data["metadata"]["testbed"] = self._testbed
+ self._msg_type = None
+
def _get_vpp_version(self, msg):
"""Called when extraction of VPP version is required.
:returns: Nothing.
"""
- if msg.message.count("return STDOUT Version:"):
+ if msg.message.count("return STDOUT Version:") or \
+ msg.message.count("VPP Version:"):
self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
group(2))
self._data["metadata"]["version"] = self._version
except KeyError:
pass
+ # TODO: Remove when definitely no NDRPDRDISC tests are used:
def _get_latency(self, msg, test_type):
"""Get the latency data from the test message.
return latency
+ def _get_ndrpdr_throughput(self, msg):
+ """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
+ message.
+
+ :param msg: The test message to be parsed.
+ :type msg: str
+ :returns: Parsed data as a dict and the status (PASS/FAIL).
+ :rtype: tuple(dict, str)
+ """
+
+ throughput = {
+ "NDR": {"LOWER": -1.0, "UPPER": -1.0},
+ "PDR": {"LOWER": -1.0, "UPPER": -1.0}
+ }
+ status = "FAIL"
+ groups = re.search(self.REGEX_NDRPDR_RATE, msg)
+
+ if groups is not None:
+ try:
+ throughput["NDR"]["LOWER"] = float(groups.group(1))
+ throughput["NDR"]["UPPER"] = float(groups.group(2))
+ throughput["PDR"]["LOWER"] = float(groups.group(3))
+ throughput["PDR"]["UPPER"] = float(groups.group(4))
+ status = "PASS"
+ except (IndexError, ValueError):
+ pass
+
+ return throughput, status
+
+ def _get_ndrpdr_latency(self, msg):
+ """Get LATENCY from the test message.
+
+ :param msg: The test message to be parsed.
+ :type msg: str
+ :returns: Parsed data as a dict and the status (PASS/FAIL).
+ :rtype: tuple(dict, str)
+ """
+
+ latency = {
+ "NDR": {
+ "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
+ "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
+ },
+ "PDR": {
+ "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
+ "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
+ }
+ }
+ status = "FAIL"
+ groups = re.search(self.REGEX_NDRPDR_LAT, msg)
+
+ if groups is not None:
+ keys = ("min", "avg", "max")
+ try:
+ latency["NDR"]["direction1"] = dict(
+ zip(keys, [float(l) for l in groups.group(1).split('/')]))
+ latency["NDR"]["direction2"] = dict(
+ zip(keys, [float(l) for l in groups.group(2).split('/')]))
+ latency["PDR"]["direction1"] = dict(
+ zip(keys, [float(l) for l in groups.group(3).split('/')]))
+ latency["PDR"]["direction2"] = dict(
+ zip(keys, [float(l) for l in groups.group(4).split('/')]))
+ status = "PASS"
+ except (IndexError, ValueError):
+ pass
+
+ return latency, status
+
def visit_suite(self, suite):
"""Implements traversing through the suite and its direct children.
:returns: Nothing.
"""
+ longname_orig = test.longname.lower()
+
+ # Check the ignore list
+ if longname_orig in self._ignore:
+ return
+
tags = [str(tag) for tag in test.tags]
test_result = dict()
- test_result["name"] = test.name.lower()
+
+ # Change the TC long name and name if defined in the mapping table
+ longname = self._mapping.get(longname_orig, None)
+ if longname is not None:
+ name = longname.split('.')[-1]
+ logging.debug("{0}\n{1}\n{2}\n{3}".format(
+ self._data["metadata"], longname_orig, longname, name))
+ else:
+ longname = longname_orig
+ name = test.name.lower()
+
+ # Remove TC number from the TC long name (backward compatibility):
+ self._test_ID = re.sub(self.REGEX_TC_NUMBER, "", longname)
+ # Remove TC number from the TC name (not needed):
+ test_result["name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
+
test_result["parent"] = test.parent.name.lower()
test_result["tags"] = tags
doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
test_result["msg"] = test.message.replace('\n', ' |br| '). \
replace('\r', '').replace('"', "'")
+ test_result["type"] = "FUNC"
test_result["status"] = test.status
+
+ if "PERFTEST" in tags:
+ # Replace info about cores (e.g. -1c-) with the info about threads
+ # and cores (e.g. -1t1c-) in the long test case names and in the
+ # test case names if necessary.
+ groups = re.search(self.REGEX_TC_NAME_OLD, self._test_ID)
+ if not groups:
+ tag_count = 0
+ for tag in test_result["tags"]:
+ groups = re.search(self.REGEX_TC_TAG, tag)
+ if groups:
+ tag_count += 1
+ tag_tc = tag
+
+ if tag_count == 1:
+ self._test_ID = re.sub(self.REGEX_TC_NAME_NEW,
+ "-{0}-".format(tag_tc.lower()),
+ self._test_ID,
+ count=1)
+ test_result["name"] = re.sub(self.REGEX_TC_NAME_NEW,
+ "-{0}-".format(tag_tc.lower()),
+ test_result["name"],
+ count=1)
+ else:
+ test_result["status"] = "FAIL"
+ self._data["tests"][self._test_ID] = test_result
+ logging.debug("The test '{0}' has no or more than one "
+ "multi-threading tags.".format(self._test_ID))
+ logging.debug("Tags: {0}".format(test_result["tags"]))
+ return
+
if test.status == "PASS" and ("NDRPDRDISC" in tags or
+ "NDRPDR" in tags or
"TCP" in tags or
- "MRR" in tags):
+ "MRR" in tags or
+ "BMRR" in tags):
+ # TODO: Remove when definitely no NDRPDRDISC tests are used:
if "NDRDISC" in tags:
- test_type = "NDR"
+ test_result["type"] = "NDR"
+ # TODO: Remove when definitely no NDRPDRDISC tests are used:
elif "PDRDISC" in tags:
- test_type = "PDR"
+ test_result["type"] = "PDR"
+ elif "NDRPDR" in tags:
+ test_result["type"] = "NDRPDR"
elif "TCP" in tags:
- test_type = "TCP"
+ test_result["type"] = "TCP"
elif "MRR" in tags:
- test_type = "MRR"
+ test_result["type"] = "MRR"
+ elif "FRMOBL" in tags or "BMRR" in tags:
+ test_result["type"] = "BMRR"
else:
+ test_result["status"] = "FAIL"
+ self._data["tests"][self._test_ID] = test_result
return
- test_result["type"] = test_type
-
- if test_type in ("NDR", "PDR"):
+ # TODO: Remove when definitely no NDRPDRDISC tests are used:
+ if test_result["type"] in ("NDR", "PDR"):
try:
rate_value = str(re.search(
self.REGEX_RATE, test.message).group(1))
int(rate_value.split('.')[0])
test_result["throughput"]["unit"] = rate_unit
test_result["latency"] = \
- self._get_latency(test.message, test_type)
- if test_type == "PDR":
+ self._get_latency(test.message, test_result["type"])
+ if test_result["type"] == "PDR":
test_result["lossTolerance"] = str(re.search(
self.REGEX_TOLERANCE, test.message).group(1))
- elif test_type in ("TCP", ):
+ elif test_result["type"] in ("NDRPDR", ):
+ test_result["throughput"], test_result["status"] = \
+ self._get_ndrpdr_throughput(test.message)
+ test_result["latency"], test_result["status"] = \
+ self._get_ndrpdr_latency(test.message)
+
+ elif test_result["type"] in ("TCP", ):
groups = re.search(self.REGEX_TCP, test.message)
- test_result["result"] = dict()
- test_result["result"]["value"] = int(groups.group(2))
- test_result["result"]["unit"] = groups.group(1)
+ test_result["result"] = int(groups.group(2))
- elif test_type in ("MRR", ):
- groups = re.search(self.REGEX_MRR, test.message)
+ elif test_result["type"] in ("MRR", "BMRR"):
test_result["result"] = dict()
- test_result["result"]["duration"] = int(groups.group(1))
- test_result["result"]["tx"] = int(groups.group(2))
- test_result["result"]["rx"] = int(groups.group(3))
- test_result["result"]["throughput"] = int(
- test_result["result"]["rx"] /
- test_result["result"]["duration"])
-
- self._test_ID = test.longname.lower()
+ groups = re.search(self.REGEX_BMRR, test.message)
+ if groups is not None:
+ items_str = groups.group(1)
+ items_float = [float(item.strip()) for item
+ in items_str.split(",")]
+ metadata = AvgStdevMetadataFactory.from_data(items_float)
+ # Next two lines have been introduced in CSIT-1179,
+ # to be removed in CSIT-1180.
+ metadata.size = 1
+ metadata.stdev = 0.0
+ test_result["result"]["receive-rate"] = metadata
+ else:
+ groups = re.search(self.REGEX_MRR, test.message)
+ test_result["result"]["receive-rate"] = \
+ AvgStdevMetadataFactory.from_data([
+ float(groups.group(3)) / float(groups.group(1)), ])
+
self._data["tests"][self._test_ID] = test_result
def end_test(self, test):
elif setup_kw.name.count("Setup performance global Variables") \
and not self._timestamp:
self._msg_type = "timestamp"
+ elif setup_kw.name.count("Setup Framework") and not self._testbed:
+ self._msg_type = "testbed"
else:
return
setup_kw.messages.visit(self)
- job name
- build number
- metadata
- - job
- - build
- - vpp version
+ (as described in ExecutionChecker documentation)
- suites
+ (as described in ExecutionChecker documentation)
- tests
- - ID: test data (as described in ExecutionChecker documentation)
+ (as described in ExecutionChecker documentation)
"""
def __init__(self, spec):
return self.data[job][build]["tests"]
- @staticmethod
- def _parse_tests(job, build, log):
+ def _parse_tests(self, job, build, log):
"""Process data from robot output.xml file and return JSON structured
data.
log.append(("ERROR", "Error occurred while parsing output.xml: "
"{0}".format(err)))
return None
- checker = ExecutionChecker(metadata)
+ checker = ExecutionChecker(metadata, self._cfg.mapping,
+ self._cfg.ignore)
result.visit(checker)
return checker.data
if success:
logs.append(("INFO", " Processing data from the build '{0}' ...".
format(build["build"])))
- data = InputData._parse_tests(job, build, logs)
+ data = self._parse_tests(job, build, logs)
if data is None:
logs.append(("ERROR", "Input data file from the job '{job}', "
"build '{build}' is damaged. Skipped.".
remove(build["file-name"])
except OSError as err:
logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
- format(build["file-name"], err)))
+ format(build["file-name"], repr(err))))
+
+ # If the time-period is defined in the specification file, remove all
+ # files which are outside the time period.
+ timeperiod = self._cfg.input.get("time-period", None)
+ if timeperiod and data:
+ now = dt.utcnow()
+ timeperiod = timedelta(int(timeperiod))
+ metadata = data.get("metadata", None)
+ if metadata:
+ generated = metadata.get("generated", None)
+ if generated:
+ generated = dt.strptime(generated, "%Y%m%d %H:%M")
+ if (now - generated) > timeperiod:
+ # Remove the data and the file:
+ state = "removed"
+ data = None
+ logs.append(
+ ("INFO",
+ " The build {job}/{build} is outdated, will be "
+ "removed".format(job=job, build=build["build"])))
+ file_name = self._cfg.input["file-name"]
+ full_name = join(
+ self._cfg.environment["paths"]["DIR[WORKING,DATA]"],
+ "{job}{sep}{build}{sep}{name}".
+ format(job=job,
+ sep=SEPARATOR,
+ build=build["build"],
+ name=file_name))
+ try:
+ remove(full_name)
+ logs.append(("INFO",
+ " The file {name} has been removed".
+ format(name=full_name)))
+ except OSError as err:
+ logs.append(("ERROR",
+ "Cannot remove the file '{0}': {1}".
+ format(full_name, repr(err))))
+
logs.append(("INFO", " Done."))
result = {
- job 1
- build 1
- - test (suite) 1 ID:
+ - test (or suite) 1 ID:
- param 1
- param 2
...
- param n
...
- - test (suite) n ID:
+ - test (or suite) n ID:
...
...
- build n
if params is None:
params = element.get("parameters", None)
+ if params:
+ params.append("type")
data = pd.Series()
try:
Code Review / csit.git / blobdiff