-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2022 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
)
REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
- REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
-
REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
- def __init__(self, metadata, mapping, ignore):
+ REGEX_SH_RUN_HOST = re.compile(
+ r'hostname=\"(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})\",hook=\"(.*)\"'
+ )
+
+ def __init__(self, metadata, mapping, ignore, process_oper):
"""Initialisation.
:param metadata: Key-value pairs to be included in "metadata" part of
:param mapping: Mapping of the old names of test cases to the new
(actual) one.
:param ignore: List of TCs to be ignored.
+ :param process_oper: If True, operational data (show run, telemetry) is
+ processed.
:type metadata: dict
:type mapping: dict
:type ignore: list
+ :type process_oper: bool
"""
# Type of message to parse out from the test messages
# Ignore list
self._ignore = ignore
+ self._process_oper = process_oper
+
# Number of PAPI History messages found:
# 0 - no message
# 1 - PAPI History of DUT1
self._conf_history_lookup_nr = 0
self._sh_run_counter = 0
+ self._telemetry_kw_counter = 0
+ self._telemetry_msg_counter = 0
# Test ID of currently processed test- the lowercase full path to the
# test
# Dictionary defining the methods used to parse different types of
# messages
self.parse_msg = {
- u"timestamp": self._get_timestamp,
u"vpp-version": self._get_vpp_version,
u"dpdk-version": self._get_dpdk_version,
u"teardown-papi-history": self._get_papi_history,
u"test-show-runtime": self._get_show_run,
- u"testbed": self._get_testbed
+ u"testbed": self._get_testbed,
+ u"test-telemetry": self._get_telemetry
}
@property
return u"Test Failed."
def _process_lat(in_str_1, in_str_2):
- """Extract min, avg, max values from latency string.
+ """Extract P50, P90 and P99 latencies or min, avg, max values from
+ latency string.
:param in_str_1: Latency string for one direction produced by robot
framework.
try:
hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
except hdrh.codec.HdrLengthException:
- return None
+ hdr_lat_1 = None
in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
try:
hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
except hdrh.codec.HdrLengthException:
- return None
+ hdr_lat_2 = None
if hdr_lat_1 and hdr_lat_2:
hdr_lat = (
hdr_lat_2.get_value_at_percentile(90.0),
hdr_lat_2.get_value_at_percentile(99.0)
)
-
if all(hdr_lat):
return hdr_lat
- return None
+ hdr_lat = (
+ int(in_list_1[0]), int(in_list_1[1]), int(in_list_1[2]),
+ int(in_list_2[0]), int(in_list_2[1]), int(in_list_2[2])
+ )
+ for item in hdr_lat:
+ if item in (-1, 4294967295, 0):
+ return None
+ return hdr_lat
try:
out_msg = (
finally:
self._msg_type = None
- def _get_timestamp(self, msg):
- """Called when extraction of timestamp is required.
-
- :param msg: Message to process.
- :type msg: Message
- :returns: Nothing.
- """
-
- self._timestamp = msg.timestamp[:14]
- self._data[u"metadata"][u"generated"] = self._timestamp
- self._msg_type = None
-
def _get_papi_history(self, msg):
"""Called when extraction of PAPI command history is required.
except (AttributeError, IndexError):
sock = u""
- runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
- replace(u"'", u'"').replace(u'b"', u'"').
- replace(u'u"', u'"').split(u":", 1)[1])
-
dut = u"dut{nr}".format(
nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
{
u"host": host,
u"socket": sock,
- u"runtime": runtime,
+ u"runtime": str(msg.message).replace(u' ', u'').
+ replace(u'\n', u'').replace(u"'", u'"').
+ replace(u'b"', u'"').replace(u'u"', u'"').
+ split(u":", 1)[1]
+ }
+ )
+
+ def _get_telemetry(self, msg):
+ """Called when extraction of VPP telemetry data is required.
+
+ :param msg: Message to process.
+ :type msg: Message
+ :returns: Nothing.
+ """
+
+ if self._telemetry_kw_counter > 1:
+ return
+ if not msg.message.count(u"# TYPE vpp_runtime_calls"):
+ return
+
+ if u"telemetry-show-run" not in \
+ self._data[u"tests"][self._test_id].keys():
+ self._data[u"tests"][self._test_id][u"telemetry-show-run"] = dict()
+
+ self._telemetry_msg_counter += 1
+ groups = re.search(self.REGEX_SH_RUN_HOST, msg.message)
+ if not groups:
+ return
+ try:
+ host = groups.group(1)
+ except (AttributeError, IndexError):
+ host = u""
+ try:
+ sock = groups.group(2)
+ except (AttributeError, IndexError):
+ sock = u""
+ runtime = {
+ u"source_type": u"node",
+ u"source_id": host,
+ u"msg_type": u"metric",
+ u"log_level": u"INFO",
+ u"timestamp": msg.timestamp,
+ u"msg": u"show_runtime",
+ u"host": host,
+ u"socket": sock,
+ u"data": list()
+ }
+ for line in msg.message.splitlines():
+ if not line.startswith(u"vpp_runtime_"):
+ continue
+ try:
+ params, value, timestamp = line.rsplit(u" ", maxsplit=2)
+ cut = params.index(u"{")
+ name = params[:cut].split(u"_", maxsplit=2)[-1]
+ labels = eval(
+ u"dict" + params[cut:].replace('{', '(').replace('}', ')')
+ )
+ labels[u"graph_node"] = labels.pop(u"name")
+ runtime[u"data"].append(
+ {
+ u"name": name,
+ u"value": value,
+ u"timestamp": timestamp,
+ u"labels": labels
+ }
+ )
+ except (TypeError, ValueError, IndexError):
+ continue
+ self._data[u'tests'][self._test_id][u'telemetry-show-run']\
+ [f"dut{self._telemetry_msg_counter}"] = copy.copy(
+ {
+ u"host": host,
+ u"socket": sock,
+ u"runtime": runtime
}
)
u"level": len(suite.longname.split(u"."))
}
- suite.keywords.visit(self)
+ suite.setup.visit(self)
def end_suite(self, suite):
"""Called when suite ends.
:returns: Nothing.
"""
if self.start_test(test) is not False:
- test.keywords.visit(self)
+ test.body.visit(self)
self.end_test(test)
def start_test(self, test):
"""
self._sh_run_counter = 0
+ self._telemetry_kw_counter = 0
+ self._telemetry_msg_counter = 0
longname_orig = test.longname.lower()
else:
test_result[u"msg"] = test.message
- if u"PERFTEST" in tags:
+ if u"PERFTEST" in tags and u"TREX" not in tags:
# Replace info about cores (e.g. -1c-) with the info about threads
# and cores (e.g. -1t1c-) in the long test case names and in the
# test case names if necessary.
- groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
- if not groups:
- tag_count = 0
- tag_tc = str()
- for tag in test_result[u"tags"]:
- groups = re.search(self.REGEX_TC_TAG, tag)
- if groups:
- tag_count += 1
- tag_tc = tag
-
- if tag_count == 1:
- self._test_id = re.sub(
- self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
- self._test_id, count=1
- )
- test_result[u"name"] = re.sub(
- self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
- test_result["name"], count=1
- )
- else:
- test_result[u"status"] = u"FAIL"
- self._data[u"tests"][self._test_id] = test_result
- logging.debug(
- f"The test {self._test_id} has no or more than one "
- f"multi-threading tags.\n"
- f"Tags: {test_result[u'tags']}"
- )
- return
+ tag_count = 0
+ tag_tc = str()
+ for tag in test_result[u"tags"]:
+ groups = re.search(self.REGEX_TC_TAG, tag)
+ if groups:
+ tag_count += 1
+ tag_tc = tag
+
+ if tag_count == 1:
+ self._test_id = re.sub(
+ self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
+ self._test_id, count=1
+ )
+ test_result[u"name"] = re.sub(
+ self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
+ test_result["name"], count=1
+ )
+ else:
+ test_result[u"status"] = u"FAIL"
+ self._data[u"tests"][self._test_id] = test_result
+ logging.debug(
+ f"The test {self._test_id} has no or more than one "
+ f"multi-threading tags.\n"
+ f"Tags: {test_result[u'tags']}"
+ )
+ return
if u"DEVICETEST" in tags:
test_result[u"type"] = u"DEVICETEST"
if test.status == u"PASS":
test_result[u"throughput"], test_result[u"status"] = \
self._get_plr_throughput(test.message)
+ elif u"LDP_NGINX" in tags:
+ test_result[u"type"] = u"LDP_NGINX"
+ test_result[u"result"], test_result[u"status"] = \
+ self._get_vsap_data(test.message, tags)
elif u"HOSTSTACK" in tags:
test_result[u"type"] = u"HOSTSTACK"
if test.status == u"PASS":
test_result[u"result"], test_result[u"status"] = \
self._get_hoststack_data(test.message, tags)
- elif u"LDP_NGINX" in tags:
- test_result[u"type"] = u"LDP_NGINX"
- test_result[u"result"], test_result[u"status"] = \
- self._get_vsap_data(test.message, tags)
- # elif u"TCP" in tags: # This might be not used
- # test_result[u"type"] = u"TCP"
- # if test.status == u"PASS":
- # groups = re.search(self.REGEX_TCP, test.message)
- # test_result[u"result"] = int(groups.group(2))
elif u"RECONF" in tags:
test_result[u"type"] = u"RECONF"
if test.status == u"PASS":
:type test_kw: Keyword
:returns: Nothing.
"""
- for keyword in test_kw.keywords:
+ for keyword in test_kw.body:
if self.start_test_kw(keyword) is not False:
self.visit_test_kw(keyword)
self.end_test_kw(keyword)
:type test_kw: Keyword
:returns: Nothing.
"""
- if test_kw.name.count(u"Show Runtime On All Duts") or \
- test_kw.name.count(u"Show Runtime Counters On All Duts") or \
- test_kw.name.count(u"Vpp Show Runtime On All Duts"):
+ if not self._process_oper:
+ return
+
+ if test_kw.name.count(u"Run Telemetry On All Duts"):
+ self._msg_type = u"test-telemetry"
+ self._telemetry_kw_counter += 1
+ elif test_kw.name.count(u"Show Runtime On All Duts"):
self._msg_type = u"test-show-runtime"
self._sh_run_counter += 1
else:
:type setup_kw: Keyword
:returns: Nothing.
"""
- for keyword in setup_kw.keywords:
+ for keyword in setup_kw.body:
if self.start_setup_kw(keyword) is not False:
self.visit_setup_kw(keyword)
self.end_setup_kw(keyword)
elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
not self._version:
self._msg_type = u"dpdk-version"
- elif setup_kw.name.count(u"Set Global Variable") \
- and not self._timestamp:
- self._msg_type = u"timestamp"
elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
self._msg_type = u"testbed"
else:
:type teardown_kw: Keyword
:returns: Nothing.
"""
- for keyword in teardown_kw.keywords:
+ for keyword in teardown_kw.body:
if self.start_teardown_kw(keyword) is not False:
self.visit_teardown_kw(keyword)
self.end_teardown_kw(keyword)
(as described in ExecutionChecker documentation)
"""
- def __init__(self, spec):
+ def __init__(self, spec, for_output):
"""Initialization.
:param spec: Specification.
+ :param for_output: Output to be generated from downloaded data.
:type spec: Specification
+ :type for_output: str
"""
# Specification:
self._cfg = spec
+ self._for_output = for_output
+
# Data store:
- self._input_data = pd.Series()
+ self._input_data = pd.Series(dtype="object")
@property
def data(self):
f"Error occurred while parsing output.xml: {repr(err)}"
)
return None
+
+ process_oper = False
+ if u"-vpp-perf-report-coverage-" in job:
+ process_oper = True
+ # elif u"-vpp-perf-report-iterative-" in job:
+ # # Exceptions for TBs where we do not have coverage data:
+ # for item in (u"-2n-icx", ):
+ # if item in job:
+ # process_oper = True
+ # break
checker = ExecutionChecker(
- metadata, self._cfg.mapping, self._cfg.ignore
+ metadata, self._cfg.mapping, self._cfg.ignore, process_oper
)
result.visit(checker)
+ checker.data[u"metadata"][u"tests_total"] = \
+ result.statistics.total.all.total
+ checker.data[u"metadata"][u"tests_passed"] = \
+ result.statistics.total.all.passed
+ checker.data[u"metadata"][u"tests_failed"] = \
+ result.statistics.total.all.failed
+ checker.data[u"metadata"][u"elapsedtime"] = result.suite.elapsedtime
+ checker.data[u"metadata"][u"generated"] = result.suite.endtime[:14]
+
return checker.data
def _download_and_parse_build(self, job, build, repeat, pid=10000):
})
if self._input_data.get(job, None) is None:
- self._input_data[job] = pd.Series()
+ self._input_data[job] = pd.Series(dtype="object")
self._input_data[job][str(build_nr)] = build_data
self._cfg.set_input_file_name(
job, build_nr, result[u"build"][u"file-name"]
})
if self._input_data.get(job, None) is None:
- self._input_data[job] = pd.Series()
+ self._input_data[job] = pd.Series(dtype="object")
self._input_data[job][str(build_nr)] = build_data
self._cfg.set_input_state(job, build_nr, u"processed")
params.extend((u"type", u"status"))
data_to_filter = data if data else element[u"data"]
- data = pd.Series()
+ data = pd.Series(dtype="object")
try:
for job, builds in data_to_filter.items():
- data[job] = pd.Series()
+ data[job] = pd.Series(dtype="object")
for build in builds:
- data[job][str(build)] = pd.Series()
+ data[job][str(build)] = pd.Series(dtype="object")
try:
data_dict = dict(
self.data[job][str(build)][data_set].items())
for test_id, test_data in data_dict.items():
if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
- data[job][str(build)][test_id] = pd.Series()
+ data[job][str(build)][test_id] = \
+ pd.Series(dtype="object")
if params is None:
for param, val in test_data.items():
data[job][str(build)][test_id][param] = val
else:
tests = include
- data = pd.Series()
+ data = pd.Series(dtype="object")
try:
for job, builds in element[u"data"].items():
- data[job] = pd.Series()
+ data[job] = pd.Series(dtype="object")
for build in builds:
- data[job][str(build)] = pd.Series()
+ data[job][str(build)] = pd.Series(dtype="object")
for test in tests:
try:
reg_ex = re.compile(str(test).lower())
if re.match(reg_ex, str(test_id).lower()):
test_data = self.data[job][
str(build)][data_set][test_id]
- data[job][str(build)][test_id] = pd.Series()
+ data[job][str(build)][test_id] = \
+ pd.Series(dtype="object")
if params is None:
for param, val in test_data.items():
data[job][str(build)][test_id]\
logging.info(u" Merging data ...")
- merged_data = pd.Series()
+ merged_data = pd.Series(dtype="object")
for builds in data.values:
for item in builds.values:
for item_id, item_data in item.items():
for dut_name, data in test_data[u"show-run"].items():
if data.get(u"runtime", None) is None:
continue
+ runtime = loads(data[u"runtime"])
try:
- threads_nr = len(data[u"runtime"][0][u"clocks"])
+ threads_nr = len(runtime[0][u"clocks"])
except (IndexError, KeyError):
continue
threads = OrderedDict(
{idx: list() for idx in range(threads_nr)})
- for item in data[u"runtime"]:
+ for item in runtime:
for idx in range(threads_nr):
if item[u"vectors"][idx] > 0:
clocks = item[u"clocks"][idx] / \
vectors_call = 0.0
if int(item[u"calls"][idx]) + int(
- item[u"vectors"][idx]) + \
- int(item[u"suspends"][idx]):
+ item[u"vectors"][idx]) + \
+ int(item[u"suspends"][idx]):
threads[idx].append([
item[u"name"],
item[u"calls"][idx],
Code Review / csit.git / blobdiff