summary |
shortlog |
log |
commit | commitdiff |
review |
tree
raw |
patch |
inline | side by side (from parent 1:
258f266)
Change-Id: I27687ed6dda3a8717538653e75ad0d61c6516c7e
Signed-off-by: Tibor Frank <tifrank@cisco.com>
success = False
try:
log.append(("INFO", " Connecting to '{0}' ...".format(url)))
success = False
try:
log.append(("INFO", " Connecting to '{0}' ...".format(url)))
response = get(url, stream=True)
code = response.status_code
response = get(url, stream=True)
code = response.status_code
log.append(("INFO", " {0}: {1}".format(code, responses[code])))
if code != codes["OK"]:
log.append(("INFO", " {0}: {1}".format(code, responses[code])))
if code != codes["OK"]:
+ url = url.replace("_info", "")
+ log.append(("INFO", " Connecting to '{0}' ...".format(url)))
+ response = get(url, stream=True)
+ code = response.status_code
+ log.append(("INFO", " {0}: {1}".format(code, responses[code])))
+ if code != codes["OK"]:
+ return False, file_name
+ file_name = file_name.replace("_info", "")
log.append(("INFO", " Downloading the file '{0}' to '{1}' ...".
format(url, file_name)))
log.append(("INFO", " Downloading the file '{0}' to '{1}' ...".
format(url, file_name)))
- file_handle = open(file_name, "wb")
- for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
- if chunk:
- file_handle.write(chunk)
- file_handle.close()
+ with open(file_name, "wb") as file_handle:
+ for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
+ if chunk:
+ file_handle.write(chunk)
success = True
except ConnectionError as err:
log.append(("ERROR", "Not possible to connect to '{0}'.".format(url)))
success = True
except ConnectionError as err:
log.append(("ERROR", "Not possible to connect to '{0}'.".format(url)))
- log.append(("DEBUG", str(err)))
+ log.append(("DEBUG", repr(err)))
except HTTPError as err:
log.append(("ERROR", "Invalid HTTP response from '{0}'.".format(url)))
except HTTPError as err:
log.append(("ERROR", "Invalid HTTP response from '{0}'.".format(url)))
- log.append(("DEBUG", str(err)))
+ log.append(("DEBUG", repr(err)))
except TooManyRedirects as err:
log.append(("ERROR", "Request exceeded the configured number "
"of maximum re-directions."))
except TooManyRedirects as err:
log.append(("ERROR", "Request exceeded the configured number "
"of maximum re-directions."))
- log.append(("DEBUG", str(err)))
+ log.append(("DEBUG", repr(err)))
except Timeout as err:
log.append(("ERROR", "Request timed out."))
except Timeout as err:
log.append(("ERROR", "Request timed out."))
- log.append(("DEBUG", str(err)))
+ log.append(("DEBUG", repr(err)))
except RequestException as err:
log.append(("ERROR", "Unexpected HTTP request exception."))
except RequestException as err:
log.append(("ERROR", "Unexpected HTTP request exception."))
- log.append(("DEBUG", str(err)))
+ log.append(("DEBUG", repr(err)))
except (IOError, ValueError, KeyError) as err:
log.append(("ERROR", "Download failed."))
except (IOError, ValueError, KeyError) as err:
log.append(("ERROR", "Download failed."))
- log.append(("DEBUG", str(err)))
+ log.append(("DEBUG", repr(err)))
log.append(("INFO", " Download finished."))
log.append(("INFO", " Download finished."))
+ return success, file_name
def _unzip_file(spec, build, pid, log):
def _unzip_file(spec, build, pid, log):
elif job.startswith("intel-dnv-"):
url = spec.environment["urls"]["URL[VIRL,DNV]"].format(release=job[-4:])
else:
elif job.startswith("intel-dnv-"):
url = spec.environment["urls"]["URL[VIRL,DNV]"].format(release=job[-4:])
else:
- raise PresentationError("No url defined for the job '{}'.".
- format(job))
+ raise PresentationError("No url defined for the job '{}'.".format(job))
file_name = spec.input["file-name"]
full_name = spec.input["download-path"]. \
format(job=job, build=build["build"], filename=file_name)
file_name = spec.input["file-name"]
full_name = spec.input["download-path"]. \
format(job=job, build=build["build"], filename=file_name)
name=file_name))
# Download the file from the defined source (Jenkins, logs.fd.io):
name=file_name))
# Download the file from the defined source (Jenkins, logs.fd.io):
- success = _download_file(url, new_name, log)
+ success, downloaded_name = _download_file(url, new_name, log)
+ if success:
+ new_name = downloaded_name
if success and new_name.endswith(".zip"):
if not is_zipfile(new_name):
if success and new_name.endswith(".zip"):
if not is_zipfile(new_name):
release=rls,
dir=spec.environment["urls"]["DIR[NEXUS]"],
file=nexus_file_name)
release=rls,
dir=spec.environment["urls"]["DIR[NEXUS]"],
file=nexus_file_name)
- success = _download_file(url, new_name, log)
+ success, new_name = _download_file(url, new_name, log)
:type msg: Message
:returns: Nothing.
"""
:type msg: Message
:returns: Nothing.
"""
- if msg.message.count("return STDOUT Thread "):
+ if msg.message.count("Thread 0 vpp_main"):
self._show_run_lookup_nr += 1
if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
self._data["tests"][self._test_ID]["show-run"] = str()
self._show_run_lookup_nr += 1
if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
self._data["tests"][self._test_ID]["show-run"] = str()
if setup_kw.name.count("Show Vpp Version On All Duts") \
and not self._version:
self._msg_type = "vpp-version"
if setup_kw.name.count("Show Vpp Version On All Duts") \
and not self._version:
self._msg_type = "vpp-version"
-
- elif setup_kw.name.count("Setup performance global Variables") \
+ elif setup_kw.name.count("Set Global Variable") \
and not self._timestamp:
self._msg_type = "timestamp"
elif setup_kw.name.count("Setup Framework") and not self._testbed:
and not self._timestamp:
self._msg_type = "timestamp"
elif setup_kw.name.count("Setup Framework") and not self._testbed:
- logging.info(" Processing the job/build: {0}: {1}".
- format(job, build["build"]))
-
logs.append(("INFO", " Processing the job/build: {0}: {1}".
format(job, build["build"])))
logs.append(("INFO", " Processing the job/build: {0}: {1}".
format(job, build["build"])))
"'{build}', or it is damaged. Skipped.".
format(job=job, build=build["build"])))
if success:
"'{build}', or it is damaged. Skipped.".
format(job=job, build=build["build"])))
if success:
- logs.append(("INFO", " Processing data from the build '{0}' ...".
+ logs.append(("INFO", " Processing data from the build '{0}' ...".
format(build["build"])))
data = self._parse_tests(job, build, logs)
if data is None:
format(build["build"])))
data = self._parse_tests(job, build, logs)
if data is None:
logs.append(("ERROR",
"Cannot remove the file '{0}': {1}".
format(full_name, repr(err))))
logs.append(("ERROR",
"Cannot remove the file '{0}': {1}".
format(full_name, repr(err))))
logs.append(("INFO", " Done."))
logs.append(("INFO", " Done."))
+ for level, line in logs:
+ if level == "INFO":
+ logging.info(line)
+ elif level == "ERROR":
+ logging.error(line)
+ elif level == "DEBUG":
+ logging.debug(line)
+ elif level == "CRITICAL":
+ logging.critical(line)
+ elif level == "WARNING":
+ logging.warning(line)
+
result = {
"data": data,
"state": state,
"job": job,
result = {
"data": data,
"state": state,
"job": job,
- "build": build,
- "logs": logs
self._cfg.set_input_state(job, build_nr, result["state"])
self._cfg.set_input_state(job, build_nr, result["state"])
- for item in result["logs"]:
- if item[0] == "INFO":
- logging.info(item[1])
- elif item[0] == "ERROR":
- logging.error(item[1])
- elif item[0] == "DEBUG":
- logging.debug(item[1])
- elif item[0] == "CRITICAL":
- logging.critical(item[1])
- elif item[0] == "WARNING":
- logging.warning(item[1])
-
del data_queue
# Terminate all workers
del data_queue
# Terminate all workers
prepare_static_content(spec)
data = InputData(spec)
prepare_static_content(spec)
data = InputData(spec)
- data.download_and_parse_data(repeat=2)
+ data.download_and_parse_data(repeat=1)
generate_tables(spec, data)
generate_plots(spec, data)
generate_tables(spec, data)
generate_plots(spec, data)
# 3n-hsw
plot-performance-trending-all-3n-hsw:
csit-vpp-perf-mrr-daily-master:
# 3n-hsw
plot-performance-trending-all-3n-hsw:
csit-vpp-perf-mrr-daily-master:
end: "lastCompletedBuild"
skip:
- 593
end: "lastCompletedBuild"
skip:
- 593
- 616
- 617
csit-dpdk-perf-mrr-weekly-master:
- 616
- 617
csit-dpdk-perf-mrr-weekly-master:
end: "lastCompletedBuild"
plot-performance-trending-vpp-3n-hsw:
csit-vpp-perf-mrr-daily-master:
end: "lastCompletedBuild"
plot-performance-trending-vpp-3n-hsw:
csit-vpp-perf-mrr-daily-master:
end: "lastCompletedBuild"
skip:
- 593
end: "lastCompletedBuild"
skip:
- 593
plot-performance-trending-dpdk-3n-hsw:
csit-dpdk-perf-mrr-weekly-master:
plot-performance-trending-dpdk-3n-hsw:
csit-dpdk-perf-mrr-weekly-master:
end: "lastCompletedBuild"
# 3n-skx
plot-performance-trending-all-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
end: "lastCompletedBuild"
# 3n-skx
plot-performance-trending-all-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
end: "lastCompletedBuild"
skip:
- 356
end: "lastCompletedBuild"
skip:
- 356
- 377
- 378
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- 377
- 378
csit-dpdk-perf-mrr-weekly-master-3n-skx:
end: "lastCompletedBuild"
skip:
- 31
plot-performance-trending-vpp-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
end: "lastCompletedBuild"
skip:
- 31
plot-performance-trending-vpp-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
end: "lastCompletedBuild"
skip:
- 356
end: "lastCompletedBuild"
skip:
- 356
plot-performance-trending-dpdk-3n-skx:
csit-dpdk-perf-mrr-weekly-master-3n-skx:
plot-performance-trending-dpdk-3n-skx:
csit-dpdk-perf-mrr-weekly-master-3n-skx:
end: "lastCompletedBuild"
skip:
- 31
end: "lastCompletedBuild"
skip:
- 31
# 2n-skx
plot-performance-trending-all-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
# 2n-skx
plot-performance-trending-all-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
end: "lastCompletedBuild"
skip:
end: "lastCompletedBuild"
skip:
+ - 398
+ - 403
+ - 404
+ - 405
csit-dpdk-perf-mrr-weekly-master-2n-skx:
csit-dpdk-perf-mrr-weekly-master-2n-skx:
end: "lastCompletedBuild"
plot-performance-trending-vpp-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
end: "lastCompletedBuild"
plot-performance-trending-vpp-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
end: "lastCompletedBuild"
skip:
end: "lastCompletedBuild"
skip:
+ - 398
+ - 403
+ - 404
+ - 405
plot-performance-trending-dpdk-2n-skx:
csit-dpdk-perf-mrr-weekly-master-2n-skx:
plot-performance-trending-dpdk-2n-skx:
csit-dpdk-perf-mrr-weekly-master-2n-skx:
end: "lastCompletedBuild"
plot-performance-trending-vpp-nfv-2n-skx:
end: "lastCompletedBuild"
plot-performance-trending-vpp-nfv-2n-skx:
- file-name: "output.xml.gz"
+ file-name: "output_info.xml.gz"
file-format: ".gz"
download-path: "{job}/{build}/archives/{filename}"
extract: "output.xml"
file-format: ".gz"
download-path: "{job}/{build}/archives/{filename}"
extract: "output.xml"
# 3n-hsw
csit-vpp-perf-mrr-daily-master:
# 3n-hsw
csit-vpp-perf-mrr-daily-master:
end: "lastCompletedBuild"
skip:
- 593
end: "lastCompletedBuild"
skip:
- 593
- 616
- 617
csit-dpdk-perf-mrr-weekly-master:
- 616
- 617
csit-dpdk-perf-mrr-weekly-master:
end: "lastCompletedBuild"
# 3n-skx
csit-vpp-perf-mrr-daily-master-3n-skx:
end: "lastCompletedBuild"
# 3n-skx
csit-vpp-perf-mrr-daily-master-3n-skx:
end: "lastCompletedBuild"
skip:
- 356
end: "lastCompletedBuild"
skip:
- 356
- 377
- 378
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- 377
- 378
csit-dpdk-perf-mrr-weekly-master-3n-skx:
end: "lastCompletedBuild"
skip:
- 31
# 2n-skx
csit-vpp-perf-mrr-daily-master-2n-skx:
end: "lastCompletedBuild"
skip:
- 31
# 2n-skx
csit-vpp-perf-mrr-daily-master-2n-skx:
end: "lastCompletedBuild"
skip:
end: "lastCompletedBuild"
skip:
+ - 398
+ - 403
+ - 404
+ - 405
csit-vpp-perf-mrr-weekly-master-2n-skx:
start: 1
end: "lastCompletedBuild"
csit-vpp-perf-mrr-weekly-master-2n-skx:
start: 1
end: "lastCompletedBuild"
- 3
- 5
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- 3
- 5
csit-dpdk-perf-mrr-weekly-master-2n-skx:
end: "lastCompletedBuild"
-
end: "lastCompletedBuild"
-
# 3n-hsw
plot-performance-trending-all-3n-hsw:
csit-vpp-perf-mrr-daily-master:
# 3n-hsw
plot-performance-trending-all-3n-hsw:
csit-vpp-perf-mrr-daily-master:
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
end: "lastCompletedBuild"
plot-performance-trending-vpp-3n-hsw:
csit-vpp-perf-mrr-daily-master:
end: "lastCompletedBuild"
plot-performance-trending-vpp-3n-hsw:
csit-vpp-perf-mrr-daily-master:
end: "lastCompletedBuild"
plot-performance-trending-dpdk-3n-hsw:
csit-dpdk-perf-mrr-weekly-master:
end: "lastCompletedBuild"
plot-performance-trending-dpdk-3n-hsw:
csit-dpdk-perf-mrr-weekly-master:
end: "lastCompletedBuild"
# 3n-skx
plot-performance-trending-all-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
end: "lastCompletedBuild"
# 3n-skx
plot-performance-trending-all-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
end: 30 # "lastCompletedBuild"
plot-performance-trending-vpp-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
end: 30 # "lastCompletedBuild"
plot-performance-trending-vpp-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
end: "lastCompletedBuild"
plot-performance-trending-dpdk-3n-skx:
csit-dpdk-perf-mrr-weekly-master-3n-skx:
end: "lastCompletedBuild"
plot-performance-trending-dpdk-3n-skx:
csit-dpdk-perf-mrr-weekly-master-3n-skx:
end: 30 # "lastCompletedBuild"
# 2n-skx
plot-performance-trending-all-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
end: 30 # "lastCompletedBuild"
# 2n-skx
plot-performance-trending-all-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
end: "lastCompletedBuild"
plot-performance-trending-vpp-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
end: "lastCompletedBuild"
plot-performance-trending-vpp-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
end: "lastCompletedBuild"
plot-performance-trending-dpdk-2n-skx:
csit-dpdk-perf-mrr-weekly-master-2n-skx:
end: "lastCompletedBuild"
plot-performance-trending-dpdk-2n-skx:
csit-dpdk-perf-mrr-weekly-master-2n-skx:
end: "lastCompletedBuild"
plot-layouts:
end: "lastCompletedBuild"
plot-layouts:
# 3n-hsw
csit-vpp-perf-mrr-daily-master:
# 3n-hsw
csit-vpp-perf-mrr-daily-master:
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
end: "lastCompletedBuild"
# 3n-skx
csit-vpp-perf-mrr-daily-master-3n-skx:
end: "lastCompletedBuild"
# 3n-skx
csit-vpp-perf-mrr-daily-master-3n-skx:
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
end: 30 # "lastCompletedBuild"
# 2n-skx
csit-vpp-perf-mrr-daily-master-2n-skx:
end: 30 # "lastCompletedBuild"
# 2n-skx
csit-vpp-perf-mrr-daily-master-2n-skx:
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
end: "lastCompletedBuild"
-
end: "lastCompletedBuild"
-