- filter the data using tags,
"""
+import copy
import re
import resource
import pandas as pd
from collections import OrderedDict
from string import replace
from os import remove
-from os.path import join
from datetime import datetime as dt
from datetime import timedelta
from json import loads
"direction1": {
"min": float,
"avg": float,
- "max": float
+ "max": float,
+ "hdrh": str
},
"direction2": {
"min": float,
"avg": float,
- "max": float
+ "max": float,
+ "hdrh": str
}
},
"PDR": {
"direction1": {
"min": float,
"avg": float,
- "max": float
+ "max": float,
+ "hdrh": str
},
"direction2": {
"min": float,
"avg": float,
- "max": float
+ "max": float,
+ "hdrh": str
}
}
}
}
}
- # TODO: Remove when definitely no NDRPDRDISC tests are used:
- # NDRPDRDISC tests:
- "ID": {
- "name": "Test name",
- "parent": "Name of the parent of the test",
- "doc": "Test documentation",
- "msg": "Test message",
- "tags": ["tag 1", "tag 2", "tag n"],
- "type": "PDR" | "NDR",
- "status": "PASS" | "FAIL",
- "throughput": { # Only type: "PDR" | "NDR"
- "value": int,
- "unit": "pps" | "bps" | "percentage"
- },
- "latency": { # Only type: "PDR" | "NDR"
- "direction1": {
- "100": {
- "min": int,
- "avg": int,
- "max": int
- },
- "50": { # Only for NDR
- "min": int,
- "avg": int,
- "max": int
- },
- "10": { # Only for NDR
- "min": int,
- "avg": int,
- "max": int
- }
- },
- "direction2": {
- "100": {
- "min": int,
- "avg": int,
- "max": int
- },
- "50": { # Only for NDR
- "min": int,
- "avg": int,
- "max": int
- },
- "10": { # Only for NDR
- "min": int,
- "avg": int,
- "max": int
- }
- }
- },
- "lossTolerance": "lossTolerance", # Only type: "PDR"
- "conf-history": "DUT1 and DUT2 VAT History"
- "show-run": "Show Run"
- },
"ID" {
# next test
}
# TODO: Remove when definitely no NDRPDRDISC tests are used:
REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
- REGEX_PLR_RATE = re.compile(r'PLRsearch lower bound::\s(\d+.\d+).*\n'
- r'PLRsearch upper bound::\s(\d+.\d+)')
+ REGEX_PLR_RATE = re.compile(r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
+ r'PLRsearch upper bound::?\s(\d+.\d+)')
REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
r'NDR_UPPER:\s(\d+.\d+).*\n'
r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
r'PDR_UPPER:\s(\d+.\d+)')
- # TODO: Remove when definitely no NDRPDRDISC tests are used:
- REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
- r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
- r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
- r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
- r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
- r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
- r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
-
- REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
- r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
- r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
-
REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
REGEX_VERSION_DPDK = re.compile(r"(DPDK version:\s*|DPDK Version:\s*)(.*)")
- REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
+ REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s(\d*).*$')
REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
r'tx\s(\d*),\srx\s(\d*)')
r' in packets per second: \[(.*)\]')
REGEX_RECONF_LOSS = re.compile(r'Packets lost due to reconfig: (\d*)')
- REGEX_RECONF_TIME = re.compile(r'Implied time lost: (\d*.\d*)')
+ REGEX_RECONF_TIME = re.compile(r'Implied time lost: (\d*.[\de-]*)')
REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
- REGEX_TC_NUMBER = re.compile(r'tc[0-9]{2}-')
+ REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
def __init__(self, metadata, mapping, ignore):
"""Initialisation.
self._data["tests"][self._test_ID]["conf-history"] = str()
else:
self._msg_type = None
- text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
+ text = re.sub("\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
"VAT command history:", "", msg.message, count=1). \
replace("\n\n", "\n").replace('\n', ' |br| ').\
replace('\r', '').replace('"', "'")
self._data["tests"][self._test_ID]["conf-history"] = str()
else:
self._msg_type = None
- text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
+ text = re.sub("\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
"PAPI command history:", "", msg.message, count=1). \
replace("\n\n", "\n").replace('\n', ' |br| ').\
replace('\r', '').replace('"', "'")
except KeyError:
pass
- # TODO: Remove when definitely no NDRPDRDISC tests are used:
- def _get_latency(self, msg, test_type):
- """Get the latency data from the test message.
-
- :param msg: Message to be parsed.
- :param test_type: Type of the test - NDR or PDR.
- :type msg: str
- :type test_type: str
- :returns: Latencies parsed from the message.
- :rtype: dict
- """
-
- if test_type == "NDR":
- groups = re.search(self.REGEX_LAT_NDR, msg)
- groups_range = range(1, 7)
- elif test_type == "PDR":
- groups = re.search(self.REGEX_LAT_PDR, msg)
- groups_range = range(1, 3)
- else:
- return {}
-
- latencies = list()
- for idx in groups_range:
- try:
- lat = [int(item) for item in str(groups.group(idx)).split('/')]
- except (AttributeError, ValueError):
- lat = [-1, -1, -1]
- latencies.append(lat)
-
- keys = ("min", "avg", "max")
- latency = {
- "direction1": {
- },
- "direction2": {
- }
- }
-
- latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
- latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
- if test_type == "NDR":
- latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
- latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
- latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
- latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
-
- return latency
-
def _get_ndrpdr_throughput(self, msg):
"""Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
message.
:returns: Parsed data as a dict and the status (PASS/FAIL).
:rtype: tuple(dict, str)
"""
-
+ latency_default = {"min": -1.0, "avg": -1.0, "max": -1.0, "hdrh": ""}
latency = {
"NDR": {
- "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
- "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
+ "direction1": copy.copy(latency_default),
+ "direction2": copy.copy(latency_default)
},
"PDR": {
- "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
- "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
+ "direction1": copy.copy(latency_default),
+ "direction2": copy.copy(latency_default)
}
}
status = "FAIL"
groups = re.search(self.REGEX_NDRPDR_LAT, msg)
+ def process_latency(in_str):
+ """Return object with parsed latency values.
+
+ TODO: Define class for the return type.
+
+ :param in_str: Input string, min/avg/max/hdrh format.
+ :type in_str: str
+ :returns: Dict with corresponding keys, except hdrh float values.
+ :rtype dict:
+ :throws IndexError: If in_str does not have enough substrings.
+ :throws ValueError: If a substring does not convert to float.
+ """
+ in_list = in_str.split('/')
+
+ rval = {
+ "min": float(in_list[0]),
+ "avg": float(in_list[1]),
+ "max": float(in_list[2]),
+ "hdrh": ""
+ }
+
+ if len(in_list) == 4:
+ rval["hdrh"] = str(in_list[3])
+
+ return rval
+
if groups is not None:
- keys = ("min", "avg", "max")
try:
- latency["NDR"]["direction1"] = dict(
- zip(keys, [float(l) for l in groups.group(1).split('/')]))
- latency["NDR"]["direction2"] = dict(
- zip(keys, [float(l) for l in groups.group(2).split('/')]))
- latency["PDR"]["direction1"] = dict(
- zip(keys, [float(l) for l in groups.group(3).split('/')]))
- latency["PDR"]["direction2"] = dict(
- zip(keys, [float(l) for l in groups.group(4).split('/')]))
+ latency["NDR"]["direction1"] = process_latency(groups.group(1))
+ latency["NDR"]["direction2"] = process_latency(groups.group(2))
+ latency["PDR"]["direction1"] = process_latency(groups.group(3))
+ latency["PDR"]["direction2"] = process_latency(groups.group(4))
status = "PASS"
except (IndexError, ValueError):
pass
("INFO",
" The build {job}/{build} is outdated, will be "
"removed".format(job=job, build=build["build"])))
- file_name = self._cfg.input["file-name"]
- full_name = join(
- self._cfg.environment["paths"]["DIR[WORKING,DATA]"],
- "{job}{sep}{build}{sep}{name}".format(
- job=job,
- sep=SEPARATOR,
- build=build["build"],
- name=file_name))
- try:
- remove(full_name)
- logs.append(("INFO",
- " The file {name} has been removed".
- format(name=full_name)))
- except OSError as err:
- logs.append(("ERROR",
- "Cannot remove the file '{0}': {1}".
- format(full_name, repr(err))))
logs.append(("INFO", " Done."))
for level, line in logs:
Code Review / csit.git / blobdiff