-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
# 3n-hsw
plot-performance-trending-all-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 300
+ start: 346
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
- start: 35
+ start: 43
end: "lastCompletedBuild"
plot-performance-trending-vpp-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 300
+ start: 346
end: "lastCompletedBuild"
plot-performance-trending-dpdk-3n-hsw:
csit-dpdk-perf-mrr-weekly-master:
- start: 35
+ start: 43
end: "lastCompletedBuild"
# 3n-skx
start: 100
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 7
+ start: 8
end: "lastCompletedBuild"
plot-performance-trending-vpp-3n-skx:
plot-performance-trending-dpdk-3n-skx:
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 7
+ start: 8
end: "lastCompletedBuild"
# 2n-skx
start: 100
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 7
+ start: 8
end: "lastCompletedBuild"
plot-performance-trending-vpp-2n-skx:
plot-performance-trending-dpdk-2n-skx:
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 7
+ start: 8
end: "lastCompletedBuild"
plot-layouts:
# 3n-hsw
csit-vpp-perf-mrr-daily-master:
- start: 300
+ start: 346
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
- start: 35
+ start: 43
end: "lastCompletedBuild"
# 3n-skx
start: 100
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 7
+ start: 8
end: "lastCompletedBuild"
# 2n-skx
start: 100
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 7
+ start: 8
end: "lastCompletedBuild"
-
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-1t1c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
long-trend-window: 180
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-2t2c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
long-trend-window: 180
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-4t4c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
long-trend-window: 180
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-2t1c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
long-trend-window: 180
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-4t2c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
long-trend-window: 180
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-8t4c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
long-trend-window: 180
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-2t1c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
long-trend-window: 180
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-4t2c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
long-trend-window: 180
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-8t4c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
long-trend-window: 180
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
xaxis = list()
for idx in data_x:
date = build_info[job_name][str(idx)][0]
- hover_str = ("date: {0}<br>"
- "value: {1:,}<br>"
- "{2}-ref: {3}<br>"
- "csit-ref: mrr-{4}-build-{5}")
+ hover_str = ("date: {date}<br>"
+ "value: {value:,}<br>"
+ "{sut}-ref: {build}<br>"
+ "csit-ref: mrr-{period}-build-{build_nr}<br>"
+ "testbed: {testbed}")
if "dpdk" in job_name:
hover_text.append(hover_str.format(
- date,
- int(in_data[idx].avg),
- "dpdk",
- build_info[job_name][str(idx)][1].
- rsplit('~', 1)[0],
- "weekly",
- idx))
+ date=date,
+ value=int(in_data[idx].avg),
+ sut="dpdk",
+ build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
+ period="weekly",
+ build_nr=idx,
+ testbed=build_info[job_name][str(idx)][2]))
elif "vpp" in job_name:
hover_text.append(hover_str.format(
- date,
- int(in_data[idx].avg),
- "vpp",
- build_info[job_name][str(idx)][1].
- rsplit('~', 1)[0],
- "daily",
- idx))
+ date=date,
+ value=int(in_data[idx].avg),
+ sut="vpp",
+ build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
+ period="daily",
+ build_nr=idx,
+ testbed=build_info[job_name][str(idx)][2]))
xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
int(date[9:11]), int(date[12:])))
# Create "build ID": "date" dict:
build_info = dict()
+ tb_tbl = spec.environment.get("testbeds", None)
for job_name, job_data in builds_dict.items():
if build_info.get(job_name, None) is None:
build_info[job_name] = OrderedDict()
for build in job_data:
+ testbed = ""
+ tb_ip = input_data.metadata(job_name, build).get("testbed", "")
+ if tb_ip and tb_tbl:
+ testbed = tb_tbl.get(tb_ip, "")
build_info[job_name][build] = (
input_data.metadata(job_name, build).get("generated", ""),
- input_data.metadata(job_name, build).get("version", "")
+ input_data.metadata(job_name, build).get("version", ""),
+ testbed
)
work_queue = multiprocessing.JoinableQueue()
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
if classification_lst:
if isnan(rel_change_last) and isnan(rel_change_long):
continue
+ if (isnan(last_avg) or
+ isnan(rel_change_last) or
+ isnan(rel_change_long)):
+ continue
tbl_lst.append(
[tbl_dict[tst_name]["name"],
- '-' if isnan(last_avg) else
round(last_avg / 1000000, 2),
- '-' if isnan(rel_change_last) else rel_change_last,
- '-' if isnan(rel_change_long) else rel_change_long,
+ rel_change_last,
+ rel_change_long,
classification_lst[-win_size:].count("regression"),
classification_lst[-win_size:].count("progression")])
generated,
input_data.metadata(job, build).get("version", ""),
build)
- except (TypeError, KeyError):
- pass # No data in output.xml for this test
+ except (TypeError, KeyError) as err:
+ logging.warning("tst_name: {} - err: {}".
+ format(tst_name, repr(err)))
+ max_fails = 0
tbl_lst = list()
for tst_data in tbl_dict.values():
fails_nr = 0
fails_last_vpp = val[2]
fails_last_csit = val[3]
if fails_nr:
+ max_fails = fails_nr if fails_nr > max_fails else max_fails
tbl_lst.append([tst_data["name"],
fails_nr,
fails_last_date,
tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
tbl_sorted = list()
- for nrf in range(table["window"], -1, -1):
+ for nrf in range(max_fails, -1, -1):
tbl_fails = [item for item in tbl_lst if item[1] == nrf]
tbl_sorted.extend(tbl_fails)
file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
from os.path import join
from datetime import datetime as dt
from datetime import timedelta
+from json import loads
from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
from input_data_files import download_and_unzip_data_file
REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
r'[\D\d]*')
- REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*)(.*)")
+ REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|"
+ r"VPP Version:\s*)(.*)")
REGEX_VERSION_DPDK = re.compile(r"(return STDOUT testpmd)([\d\D\n]*)"
r"(RTE Version: 'DPDK )(.*)(')")
# Timestamp
self._timestamp = None
+ # Testbed. The testbed is identified by TG node IP address.
+ self._testbed = None
+
# Mapping of TCs long names
self._mapping = mapping
"vpp-version": self._get_vpp_version,
"dpdk-version": self._get_dpdk_version,
"teardown-vat-history": self._get_vat_history,
- "test-show-runtime": self._get_show_run
+ "test-show-runtime": self._get_show_run,
+ "testbed": self._get_testbed
}
@property
"""
return self._data
+ def _get_testbed(self, msg):
+ """Called when extraction of testbed IP is required.
+ The testbed is identified by TG node IP address.
+
+ :param msg: Message to process.
+ :type msg: Message
+ :returns: Nothing.
+ """
+
+ if msg.message.count("Arguments:"):
+ message = str(msg.message).replace(' ', '').replace('\n', '').\
+ replace("'", '"').replace('b"', '"').\
+ replace("honeycom", "honeycomb")
+ message = loads(message[11:-1])
+ try:
+ self._testbed = message["TG"]["host"]
+ except (KeyError, ValueError):
+ pass
+ finally:
+ self._data["metadata"]["testbed"] = self._testbed
+ self._msg_type = None
+
def _get_vpp_version(self, msg):
"""Called when extraction of VPP version is required.
:returns: Nothing.
"""
- if msg.message.count("return STDOUT Version:"):
+ if msg.message.count("return STDOUT Version:") or \
+ msg.message.count("VPP Version:"):
self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
group(2))
self._data["metadata"]["version"] = self._version
elif setup_kw.name.count("Setup performance global Variables") \
and not self._timestamp:
self._msg_type = "timestamp"
+ elif setup_kw.name.count("Setup Framework") and not self._testbed:
+ self._msg_type = "testbed"
else:
return
setup_kw.messages.visit(self)
# All directories MUST be defined in "paths" section.
- "DIR[BUILD,HTML]"
+ testbeds:
+ "10.30.51.45": "LF-2n-SKX-21"
+ "10.30.51.53": "LF-2n-SKX-22"
+ "10.30.51.55": "LF-2n-SKX-23"
+ "10.30.51.57": "LF-2n-SKX-24"
+ "10.30.51.16": "LF-3n-HSW-01"
+ "10.30.51.20": "LF-3n-HSW-02"
+ "10.30.51.24": "LF-3n-HSW-03"
+ "10.30.51.48": "LF-3n-SKX-31"
+ "10.30.51.60": "LF-3n-SKX-32"
+
-
type: "configuration"
# 3n-hsw
plot-performance-trending-all-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 300
+ start: 346
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
- start: 50
+ start: 43
end: "lastCompletedBuild"
plot-performance-trending-vpp-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 300
+ start: 346
end: "lastCompletedBuild"
plot-performance-trending-dpdk-3n-hsw:
csit-dpdk-perf-mrr-weekly-master:
- start: 50
+ start: 43
end: "lastCompletedBuild"
# 3n-skx
plot-performance-trending-all-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 50
+ start: 100
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 3
+ start: 8
end: "lastCompletedBuild"
plot-performance-trending-vpp-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 50
+ start: 100
end: "lastCompletedBuild"
plot-performance-trending-dpdk-3n-skx:
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 3
+ start: 8
end: "lastCompletedBuild"
# 2n-skx
plot-performance-trending-all-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 50
+ start: 100
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 3
+ start: 8
end: "lastCompletedBuild"
plot-performance-trending-vpp-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 50
+ start: 100
end: "lastCompletedBuild"
plot-performance-trending-dpdk-2n-skx:
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 3
+ start: 8
end: "lastCompletedBuild"
plot-layouts:
# 3n-hsw
csit-vpp-perf-mrr-daily-master:
- start: 300
+ start: 346
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
- start: 50
+ start: 43
end: "lastCompletedBuild"
# 3n-skx
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 50
+ start: 100
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 3
+ start: 8
end: "lastCompletedBuild"
# 2n-skx
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 50
+ start: 100
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 3
+ start: 8
end: "lastCompletedBuild"
-
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-1t1c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-2t2c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-4t4c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-2t1c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-4t2c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-8t4c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-2t1c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-4t2c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
# This test is "ndrdisc" test and was improperly tagged. It was fixed
# but it remains in the old output.xml files.
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-8t4c-eth-l2bdscale1mmaclrn-ndrdisc"
- outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
except KeyError:
self._specification["environment"]["build-dirs"] = None
+ try:
+ self._specification["environment"]["testbeds"] = \
+ self._cfg_yaml[idx]["testbeds"]
+ except KeyError:
+ self._specification["environment"]["testbeds"] = None
+
logging.info("Done.")
def _parse_configuration(self):
Code Review / csit.git / commitdiff