{
- "$id": "https://fd.io/FIXME/CSIT/UTI/test_case/info/1.2.0",
+ "$id": "https://fd.io/FIXME/CSIT/UTI/test_case/info/1.3.0",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "Schema for output of test case.",
"allOf": [
}
},
"required": [
+ "type",
"loss",
"aggregate_rate"
]
+ },
+ {
+ "description": "Result type HOSTSTACK case.",
+ "additionalProperties": false,
+ "properties": {
+ "type": {
+ "const": "hoststack"
+ },
+ "bandwidth": {
+ "description": "Goodput measured in bits per second.",
+ "$ref": "#/$defs/types/bandwidth"
+ },
+ "completed_requests": {
+ "description": "Number of completed requests.",
+ "$ref": "#/$defs/types/count_requests"
+ },
+ "failed_requests": {
+ "description": "Number of failed requests.",
+ "$ref": "#/$defs/types/count_requests"
+ },
+ "retransmits": {
+ "description": "Number of retransmits.",
+ "$ref": "#/$defs/types/count_packets"
+ },
+ "latency": {
+ "description": "Value and unit of latency.",
+ "$ref": "#/$defs/types/value_with_unit"
+ },
+ "duration": {
+ "description": "The relative time difference (in seconds) between program start and end.",
+ "$ref": "#/$defs/types/time_quantity"
+ },
+ "rate": {
+ "description": "RPS or CPS rate, with corresponding unit, as reported by TG.",
+ "$ref": "#/$defs/types/rate_without_bandwidth"
+ }
+ },
+ "required": [
+ "type",
+ "bandwidth"
+ ]
}
]
}
"version": {
"description": "CSIT model version (semver format) the exporting code adhered to.",
"type": "string",
- "const": "1.2.0"
+ "const": "1.3.0"
}
},
"required": [
"maxItems": 0
},
"rate_unit": {
- "description": "Packets per second (pps) or connections per second (cps).",
+ "description": "Packets per second (pps), connections per second (cps), requests per second (rps).",
"type": "string",
"enum": [
"pps",
- "cps"
+ "cps",
+ "rps"
]
},
"bandwidth_unit": {
}
]
},
+ "count_requests": {
+ "description": "Type, for counting requests.",
+ "allOf": [
+ {
+ "$ref": "#/$defs/types/value_with_unit"
+ },
+ {
+ "properties": {
+ "value": {
+ "description": "A number of requests of interest."
+ },
+ "unit": {
+ "description": "Unit suitable for displaying request counts.",
+ "enum": [
+ "requests"
+ ]
+ }
+ }
+ }
+ ]
+ },
"time_quantity": {
"description": "Reusable type, for various time quantites.",
"allOf": [
-# Copyright (c) 2022 Cisco and/or its affiliates.
+# Copyright (c) 2023 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
---
-$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.2.0
+$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.3.0
$schema: https://json-schema.org/draft/2020-12/schema
description: >-
Schema for output of test case.
packet rate.
$ref: "#/$defs/types/packet_with_time"
required:
+ - type
- loss
- aggregate_rate
+ - description: >-
+ Result type HOSTSTACK case.
+ additionalProperties: false
+ properties:
+ type:
+ const: hoststack
+ bandwidth:
+ description: >-
+ Goodput measured in bits per second.
+ $ref: "#/$defs/types/bandwidth"
+ completed_requests:
+ description: >-
+ Number of completed requests.
+ $ref: "#/$defs/types/count_requests"
+ failed_requests:
+ description: >-
+ Number of failed requests.
+ $ref: "#/$defs/types/count_requests"
+ retransmits:
+ description: >-
+ Number of retransmits.
+ $ref: "#/$defs/types/count_packets"
+ latency:
+ description: >-
+ Value and unit of latency.
+ $ref: "#/$defs/types/value_with_unit"
+ duration:
+ description: >-
+ The relative time difference (in seconds)
+ between program start and end.
+ $ref: "#/$defs/types/time_quantity"
+ rate:
+ description: >-
+ RPS or CPS rate, with corresponding unit, as
+ reported by TG.
+ $ref: "#/$defs/types/rate_without_bandwidth"
+ required:
+ - type
+ - bandwidth
start_time:
description: >-
UTC date and time in RFC 3339 format, specifying calendar time
CSIT model version (semver format)
the exporting code adhered to.
type: string
- const: 1.2.0
+ const: 1.3.0
required:
- duration
- dut_type
maxItems: 0
rate_unit:
description: >-
- Packets per second (pps) or connections per second (cps).
+ Packets per second (pps),
+ connections per second (cps),
+ requests per second (rps).
type: string
enum:
- pps
- cps
+ - rps
bandwidth_unit:
description: >-
Unit of measurement for bandwidth values.
Unit suitable for displaying packet counts.
enum:
- packets
+ count_requests:
+ description: >-
+ Type, for counting requests.
+ allOf:
+ - $ref: "#/$defs/types/value_with_unit"
+ - properties:
+ value:
+ description: >-
+ A number of requests of interest.
+ unit:
+ description: >-
+ Unit suitable for displaying request counts.
+ enum:
+ - requests
time_quantity:
description: >-
Reusable type, for various time quantites.
- properties:
value:
description: >-
- Bandwidth value computed
- from the corresponding rate.
+ Bandwidth value computed from the corresponding
+ rate.
unit:
$ref: "#/$defs/types/bandwidth_unit"
rate_with_bandwidth:
+++ /dev/null
-
-Add description with link to methodology for MRR, NDRPDR and SOAK.
-
-Add multiplicity field to MRR result, so PAL can detect incomplete samples.
-
-Add link explaining our L1 bandwidth calculation.
-
-Add a link to URL explaining how to decode the hdrh data.
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2023 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Utility to convert from .schema.yaml to .schema.json.
-
-TODO: Read the input file name from command line argument.
-"""
+"""Utility to convert from .schema.yaml to .schema.json."""
import glob
import json
for filename in glob.glob(u"*.schema.yaml"):
name = filename[:-5]
with open(f"{name}.yaml", u"r") as fin, open(f"{name}.json", u"w") as fout:
- json.dump(yaml.load(fin.read()), fout, indent=2)
+ json.dump(yaml.safe_load(fin.read()), fout, indent=2)
..
- Copyright (c) 2022 Cisco and/or its affiliates.
+ Copyright (c) 2023 Cisco and/or its affiliates.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at:
Version
~~~~~~~
-This document is valid for CSIT model version 1.2.0.
+This document is valid for CSIT model version 1.3.0.
It is recommended to use semantic versioning: https://semver.org/
That means, if the new model misses a field present in the old model,
-# Copyright (c) 2022 Cisco and/or its affiliates.
+# Copyright (c) 2023 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
"""Constants used in CSIT."""
# Version for CSIT data model. See docs/model/.
- MODEL_VERSION = u"1.2.0"
+ MODEL_VERSION = u"1.3.0"
# Global off-switch in case JSON export is large or slow.
EXPORT_JSON = get_optimistic_bool_from_env(u"EXPORT_JSON")
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2023 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
from robot.api import logger
from resources.libraries.python.Constants import Constants
-from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
-from resources.libraries.python.PapiExecutor import PapiSocketExecutor
from resources.libraries.python.DUTSetup import DUTSetup
+from resources.libraries.python.model.ExportResult import (
+ export_hoststack_results
+)
+from resources.libraries.python.PapiExecutor import PapiSocketExecutor
+from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
class HoststackUtil():
"""Utilities for Host Stack tests."""
ip_address = f" {iperf3_attributes[u'ip_address']}" if u"ip_address" \
in iperf3_attributes else u""
iperf3_cmd[u"name"] = u"iperf3"
- # TODO: Use OptionString library.
iperf3_cmd[u"args"] = f"--{iperf3_attributes[u'role']}{ip_address} " \
f"--interval 0{json_results} " \
f"--version{iperf3_attributes[u'ip_version']}"
cmd = f"sh -c 'strace -qqe trace=none -p {program_pid}'"
exec_cmd(node, cmd, sudo=True)
# Wait a bit for stdout/stderr to be flushed to log files
- # TODO: see if sub-second sleep works e.g. sleep(0.1)
sleep(1)
@staticmethod
f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
f"pkts/drop\n"
- # TODO: Incorporate show error stats into results analysis
test_results += \
f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \
f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
if u"JSON stats" in program_stdout and \
u'"has_failed": "0"' in program_stdout:
json_start = program_stdout.find(u"{")
- #TODO: Fix parsing once vpp_echo produces valid
- # JSON output. Truncate for now.
json_end = program_stdout.find(u',\n "closing"')
json_results = f"{program_stdout[json_start:json_end]}\n}}"
program_json = json.loads(json_results)
+ export_hoststack_results(
+ bandwidth=program_json["rx_bits_per_second"],
+ duration=program_json["time"]
+ )
else:
test_results += u"Invalid test data output!\n" + program_stdout
return (True, test_results)
test_results += program_stdout
iperf3_json = json.loads(program_stdout)
program_json = iperf3_json[u"intervals"][0][u"sum"]
+ export_hoststack_results(
+ bandwidth=program_json["bits_per_second"],
+ duration=program_json["seconds"],
+ retransmits=program_json["retransmits"]
+ )
else:
test_results += u"Unknown HostStack Test Program!\n" + \
program_stdout
-# Copyright (c) 2022 Cisco and/or its affiliates.
+# Copyright (c) 2023 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
from resources.libraries.python.model.util import descend, get_export_data
-def export_dut_type_and_version(dut_type=u"unknown", dut_version=u"unknown"):
+def export_dut_type_and_version(dut_type="unknown", dut_version="unknown"):
"""Export the arguments as dut type and version.
Robot tends to convert "none" into None, hence the unusual default values.
:type dut_version: Optiona[str]
:raises RuntimeError: If value is neither in argument not robot variable.
"""
- if dut_type == u"unknown":
- dut_type = BuiltIn().get_variable_value(u"\\${DUT_TYPE}", u"unknown")
- if dut_type == u"unknown":
- raise RuntimeError(u"Dut type not provided.")
+ if dut_type == "unknown":
+ dut_type = BuiltIn().get_variable_value("\\${DUT_TYPE}", "unknown")
+ if dut_type == "unknown":
+ raise RuntimeError("Dut type not provided.")
else:
# We want to set a variable in higher level suite setup
# to be available to test setup several levels lower.
BuiltIn().set_suite_variable(
- u"\\${DUT_TYPE}", dut_type, u"children=True"
+ "\\${DUT_TYPE}", dut_type, "children=True"
)
- if dut_version == u"unknown":
+ if dut_version == "unknown":
dut_version = BuiltIn().get_variable_value(
- u"\\${DUT_VERSION}", u"unknown"
+ "\\${DUT_VERSION}", "unknown"
)
- if dut_type == u"unknown":
- raise RuntimeError(u"Dut version not provided.")
+ if dut_type == "unknown":
+ raise RuntimeError("Dut version not provided.")
else:
BuiltIn().set_suite_variable(
- u"\\${DUT_VERSION}", dut_version, u"children=True"
+ "\\${DUT_VERSION}", dut_version, "children=True"
)
data = get_export_data()
- data[u"dut_type"] = dut_type.lower()
- data[u"dut_version"] = dut_version
+ data["dut_type"] = dut_type.lower()
+ data["dut_version"] = dut_version
-def export_tg_type_and_version(tg_type=u"unknown", tg_version=u"unknown"):
+def export_tg_type_and_version(tg_type="unknown", tg_version="unknown"):
"""Export the arguments as tg type and version.
Robot tends to convert "none" into None, hence the unusual default values.
:type tg_version: Optiona[str]
:raises RuntimeError: If value is neither in argument not robot variable.
"""
- if tg_type == u"unknown":
- tg_type = BuiltIn().get_variable_value(u"\\${TG_TYPE}", u"unknown")
- if tg_type == u"unknown":
- raise RuntimeError(u"TG type not provided.")
+ if tg_type == "unknown":
+ tg_type = BuiltIn().get_variable_value("\\${TG_TYPE}", "unknown")
+ if tg_type == "unknown":
+ raise RuntimeError("TG type not provided!")
else:
# We want to set a variable in higher level suite setup
# to be available to test setup several levels lower.
BuiltIn().set_suite_variable(
- u"\\${TG_TYPE}", tg_type, u"children=True"
+ "\\${TG_TYPE}", tg_type, "children=True"
)
- if tg_version == u"unknown":
+ if tg_version == "unknown":
tg_version = BuiltIn().get_variable_value(
- u"\\${TG_VERSION}", u"unknown"
+ "\\${TG_VERSION}", "unknown"
)
- if tg_type == u"unknown":
- raise RuntimeError(u"TG version not provided.")
+ if tg_type == "unknown":
+ raise RuntimeError("TG version not provided!")
else:
BuiltIn().set_suite_variable(
- u"\\${TG_VERSION}", tg_version, u"children=True"
+ "\\${TG_VERSION}", tg_version, "children=True"
)
data = get_export_data()
- data[u"tg_type"] = tg_type.lower()
- data[u"tg_version"] = tg_version
+ data["tg_type"] = tg_type.lower()
+ data["tg_version"] = tg_version
def append_mrr_value(mrr_value, unit):
if not unit:
return
data = get_export_data()
- data[u"result"][u"type"] = u"mrr"
- rate_node = descend(descend(data[u"result"], u"receive_rate"), "rate")
- rate_node[u"unit"] = str(unit)
- values_list = descend(rate_node, u"values", list)
+ data["result"]["type"] = "mrr"
+ rate_node = descend(descend(data["result"], "receive_rate"), "rate")
+ rate_node["unit"] = str(unit)
+ values_list = descend(rate_node, "values", list)
values_list.append(float(mrr_value))
"""
value = float(value)
text = str(text).lower()
- result_type = u"soak" if u"plrsearch" in text else u"ndrpdr"
- upper_or_lower = u"upper" if u"upper" in text else u"lower"
- ndr_or_pdr = u"ndr" if u"ndr" in text else u"pdr"
+ result_type = "soak" if "plrsearch" in text else "ndrpdr"
+ upper_or_lower = "upper" if "upper" in text else "lower"
+ ndr_or_pdr = "ndr" if "ndr" in text else "pdr"
- result_node = get_export_data()[u"result"]
- result_node[u"type"] = result_type
+ result_node = get_export_data()["result"]
+ result_node["type"] = result_type
rate_item = dict(rate=dict(value=value, unit=unit))
if bandwidth:
- rate_item[u"bandwidth"] = dict(value=float(bandwidth), unit=u"bps")
- if result_type == u"soak":
- descend(result_node, u"critical_rate")[upper_or_lower] = rate_item
+ rate_item["bandwidth"] = dict(value=float(bandwidth), unit="bps")
+ if result_type == "soak":
+ descend(result_node, "critical_rate")[upper_or_lower] = rate_item
return
descend(result_node, ndr_or_pdr)[upper_or_lower] = rate_item
:type whichward: str
:latency_string: str
"""
- l_min, l_avg, l_max, l_hdrh = latency_string.split(u"/", 3)
+ l_min, l_avg, l_max, l_hdrh = latency_string.split("/", 3)
whichward_node = descend(result_node, f"latency_{whichward}")
percent_node = descend(whichward_node, f"pdr_{percent}")
- percent_node[u"min"] = int(l_min)
- percent_node[u"avg"] = int(l_avg)
- percent_node[u"max"] = int(l_max)
- percent_node[u"hdrh"] = l_hdrh
- percent_node[u"unit"] = u"us"
+ percent_node["min"] = int(l_min)
+ percent_node["avg"] = int(l_avg)
+ percent_node["max"] = int(l_max)
+ percent_node["hdrh"] = l_hdrh
+ percent_node["unit"] = "us"
def export_ndrpdr_latency(text, latency):
:type text: str
:type latency: 1-tuple or 2-tuple of str
"""
- result_node = get_export_data()[u"result"]
+ result_node = get_export_data()["result"]
percent = 0
- if u"90" in text:
+ if "90" in text:
percent = 90
- elif u"50" in text:
+ elif "50" in text:
percent = 50
- elif u"10" in text:
+ elif "10" in text:
percent = 10
- _add_latency(result_node, percent, u"forward", latency[0])
+ _add_latency(result_node, percent, "forward", latency[0])
# Else TRex does not support latency measurement for this traffic profile.
if len(latency) < 2:
return
- _add_latency(result_node, percent, u"reverse", latency[1])
+ _add_latency(result_node, percent, "reverse", latency[1])
def export_reconf_result(packet_rate, packet_loss, bandwidth):
- """Export the results from a reconf test.
+ """Export the RECONF type results.
+
+ Result type is set to reconf.
:param packet_rate: Aggregate offered load in packets per second.
:param packet_loss: How many of the packets were dropped or unsent.
)
+def export_hoststack_results(
+ bandwidth, rate=None, rate_unit=None, latency=None,
+ failed_requests=None, completed_requests=None, retransmits=None,
+ duration=None
+):
+ """Export the HOSTSTACK type results.
+
+ Result type is set to hoststack.
+
+ :param bandwidth: Measured transfer rate using bps as a unit.
+ :param rate: Resulting rate measured by the test. [Optional]
+ :param rate_unit: CPS or RPS. [Optional]
+ :param latency: Measure latency. [Optional]
+ :param failed_requests: Number of failed requests. [Optional]
+ :param completed_requests: Number of completed requests. [Optional]
+ :param retransmits: Retransmitted TCP packets. [Optional]
+ :param duration: Measurment duration. [Optional]
+ :type bandwidth: float
+ :type rate: float
+ :type rate_unit: str
+ :type latency: float
+ :type failed_requests: int
+ :type completed_requests: int
+ :type retransmits: int
+ :type duration: float
+ """
+ result_node = get_export_data()["result"]
+ result_node["type"] = "hoststack"
+
+ result_node["bandwidth"] = dict(unit="bps", value=bandwidth)
+ if rate is not None:
+ result_node["rate"] = \
+ dict(unit=rate_unit, value=rate)
+ if latency is not None:
+ result_node["latency"] = \
+ dict(unit="ms", value=latency)
+ if failed_requests is not None:
+ result_node["failed_requests"] = \
+ dict(unit="requests", value=failed_requests)
+ if completed_requests is not None:
+ result_node["completed_requests"] = \
+ dict(unit="requests", value=completed_requests)
+ if retransmits is not None:
+ result_node["retransmits"] = \
+ dict(unit="packets", value=retransmits)
+ if duration is not None:
+ result_node["duration"] = \
+ dict(unit="s", value=duration)
+
+
def append_telemetry(telemetry_item):
"""Append telemetry entry to proper place so it is dumped into json.
:type telemetry_item: str
"""
data = get_export_data()
- data[u"telemetry"].append(telemetry_item)
+ data["telemetry"].append(telemetry_item)
#!/usr/bin/env python3
-# Copyright (c) 2021 Intel and/or its affiliates.
+# Copyright (c) 2023 Intel and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
# Output results.
print(f"Transfer Rate: {round(info_list[6], 2)} [Kbytes/sec]")
print(f"Latency: {round(info_list[4] / 8, 2)} ms")
- print(f"Connection {mode} rate:{round(info_list[3], 2)} per sec")
+ print(f"Connection {mode} rate: {round(info_list[3], 2)} per sec")
print(f"Total data transferred: {round(info_list[2])} bytes")
print(f"Completed requests: {round(info_list[0])} ")
print(f"Failed requests: {round(info_list[1])} ")
-# Copyright (c) 2022 Intel and/or its affiliates.
+# Copyright (c) 2023 Intel and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
"""ab implementation into CSIT framework."""
-from robot.api import logger
-from resources.libraries.python.topology import NodeType
from resources.libraries.python.Constants import Constants
-from resources.libraries.python.ssh import exec_cmd_no_error
+from resources.libraries.python.model.ExportResult import (
+ export_hoststack_results
+)
from resources.libraries.python.OptionString import OptionString
+from resources.libraries.python.ssh import exec_cmd_no_error
+from resources.libraries.python.topology import NodeType
class ABTools:
port=port,
mode=rps_cps,
)
- stdout, _ = exec_cmd_no_error(tg_node, cmd, timeout=180, sudo=True,
- message=u"ab runtime error!")
- log_msg = ABTools._parse_ab_output(stdout, rps_cps, tls_tcp)
-
- logger.info(log_msg)
-
- return log_msg
-
- @staticmethod
- def _parse_ab_output(msg, rps_cps, tls_tcp):
- """Parse the ab stdout with the results.
-
- :param msg: Ab Stdout.
- :param rps_cps: RPS or CPS.
- :param tls_tcp: TLS or TCP.
- :type msg: str
- :type rps_cps: str
- :type tls_tcp: str
- :return: Message with measured data.
- :rtype: str
- """
-
- msg_lst = msg.splitlines(keepends=False)
-
- total_cps = u""
- latency = u""
- processing = u""
- complete_req = u""
- failed_req = u""
- total_bytes = u""
- rate = u""
-
- if tls_tcp == u"tls":
- log_msg = u"\nMeasured HTTPS values:\n"
- else:
- log_msg = u"\nMeasured HTTP values:\n"
+ stdout, _ = exec_cmd_no_error(
+ tg_node, cmd, timeout=180, sudo=True, message=u"ab runtime error!"
+ )
- for line in msg_lst:
+ rate_unit = rps_cps
+ rate = None
+ bandwidth = None
+ latency = None
+ completed_requests = None
+ failed_requests = None
+ for line in stdout.splitlines():
if f"Connection {rps_cps} rate:" in line:
- # rps (cps)
- total_cps = line + u"\n"
+ rate = float(line.split(u" ")[3])
elif u"Transfer Rate:" in line:
- # Rate
- rate = line + u"\n"
+ bandwidth = float(line.split(u" ")[2]) * 8000
elif u"Latency:" in line:
- # Latency
- latency = line + u"\n"
- elif u"Total data transferred" in line:
- total_bytes = line + u"\n"
- elif u"Completed requests" in line:
- complete_req = line + u"\n"
+ latency = float(line.split(u" ")[1])
+ elif u"Completed requests:" in line:
+ completed_requests = int(line.split(u" ")[2])
elif u"Failed requests" in line:
- failed_req = line + u"\n"
+ failed_requests = int(line.split(u" ")[2])
- log_msg += rate
- log_msg += latency
- log_msg += processing
- log_msg += complete_req
- log_msg += failed_req
- log_msg += total_bytes
- log_msg += total_cps
+ export_hoststack_results(
+ rate, rate_unit, bandwidth, latency, failed_requests,
+ completed_requests
+ )
- return log_msg
+ return stdout
Code Review / csit.git / commitdiff