{
- "$id": "https://fd.io/FIXME/CSIT/UTI/test_case/info/1.1.0",
+ "$id": "https://fd.io/FIXME/CSIT/UTI/test_case/info/1.2.0",
"$schema": "https://json-schema.org/draft/2020-12/schema",
- "description": "Schema for info output of test case.",
+ "description": "Schema for output of test case.",
"allOf": [
{
"description": "The main structure, without conditional relations between fields yet.",
"additionalProperties": false,
"properties": {
"duration": {
- "description": "A derived quantity, present only in info output files. Difference between start_time and end_time, in seconds.",
+ "description": "A derived quantity. Difference between start_time and end_time, in seconds.",
"$ref": "#/$defs/types/nonnegative_number"
},
"dut_type": {
"type",
"critical_rate"
]
+ },
+ {
+ "description": "Result type RECONF case.",
+ "additionalProperties": false,
+ "properties": {
+ "type": {
+ "const": "reconf"
+ },
+ "aggregate_rate": {
+ "description": "Load used when reconfiguring, found as NDR lower bound. This is an aggregate (bidirectional) rate. Note that packets which the Traffic Generator did not send, are also counted as lost packets.",
+ "$ref": "#/$defs/types/rate_with_bandwidth"
+ },
+ "loss": {
+ "description": "Number of packets lost during reconfiguration, with the time that equals packet loss divided by packet rate.",
+ "$ref": "#/$defs/types/packet_with_time"
+ }
+ },
+ "required": [
+ "loss",
+ "aggregate_rate"
+ ]
}
]
}
"type": "string"
},
"test_id": {
- "description": "A derived quantity, present only in info output files. It is the most complete and unique identifier for a test case. This property has a value, of the following form: {suite_name}.{test_name} Here, suite name comes from SUITE_NAME robot variable, test name comes from TEST_NAME robot variable, but both are converted to lower case, and spaces are replaced by underscores.",
+ "description": "A derived quantity. It is the most complete and unique identifier for a test case. This property has a value, of the following form: {suite_name}.{test_name} Here, suite name comes from SUITE_NAME robot variable, test name comes from TEST_NAME robot variable, but both are converted to lower case, and spaces are replaced by underscores.",
"type": "string",
"minLength": 3
},
"test_name_long": {
- "description": "A derived quantity, present only in info output files. This property has a value, of the following form: {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part} Here, suite part is very similar to suite tag, but additionally may contain a prefix describing NIC driver used (if it is not the default one, drv_vfio_pci for VPP tests). Any space is replaced by underscore and letters are lower case.",
+ "description": "A derived quantity. This property has a value, of the following form: {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part} Here, suite part is very similar to suite tag, but additionally may contain a prefix describing NIC driver used (if it is not the default one, drv_vfio_pci for VPP tests). Any space is replaced by underscore and letters are lower case.",
"type": "string",
"minLength": 3
},
"test_name_short": {
- "description": "A derived quantity, present only in info output files. This property has a value very similar to suite tag, but additionally may contain a prefix describing NIC driver used (if it is not the default one, drv_vfio_pci for VPP tests). Any space is replaced by underscore and letters are lower case.",
+ "description": "A derived quantity. This property has a value very similar to suite tag, but additionally may contain a prefix describing NIC driver used (if it is not the default one, drv_vfio_pci for VPP tests). Any space is replaced by underscore and letters are lower case.",
"type": "string",
"minLength": 3
},
"test_type": {
- "description": "A derived quantity, present only in info output files. Test type identifier, PAL uses it to group similar tests, e.g. for comparison tables. Ideally, this information should be parseable from test name, but the current naming scheme is not simple/consistent enough. The current implementation queries the robot test tags. The resulting value is frequently identical to result type, but this schema version does not require any relation there, as PAL may want to group tests differently.",
+ "description": "A derived quantity. Test type identifier, PAL uses it to group similar tests, e.g. for comparison tables. Ideally, this information should be parseable from test name, but the current naming scheme is not simple/consistent enough. The current implementation queries the robot test tags. The resulting value is frequently identical to result type, but this schema version does not require any relation there, as PAL may want to group tests differently.",
"type": "string",
"enum": [
"device",
"version": {
"description": "CSIT model version (semver format) the exporting code adhered to.",
"type": "string",
- "const": "1.1.0"
+ "const": "1.2.0"
}
},
"required": [
"bps"
]
},
+ "count_packets": {
+ "description": "Type, for counting packets.",
+ "allOf": [
+ {
+ "$ref": "#/$defs/types/value_with_unit"
+ },
+ {
+ "properties": {
+ "value": {
+ "description": "A number of packets of interest."
+ },
+ "unit": {
+ "description": "Unit suitable for displaying packet counts.",
+ "enum": [
+ "packets"
+ ]
+ }
+ }
+ }
+ ]
+ },
+ "time_quantity": {
+ "description": "Reusable type, for various time quantites.",
+ "allOf": [
+ {
+ "$ref": "#/$defs/types/value_with_unit"
+ },
+ {
+ "properties": {
+ "value": {
+ "description": "Unless specified otherwise, this is a duration between two events."
+ },
+ "unit": {
+ "description": "Only seconds are the unit supported for time quantities.",
+ "enum": [
+ "s"
+ ]
+ }
+ }
+ }
+ ]
+ },
"value_with_unit": {
"description": "Reusable composite type, value together with its unit of measurement.",
"type": "object",
"rate"
]
},
+ "packet_with_time": {
+ "description": "Reusable composite type, joining packet count with the time quantity.",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "packet": {
+ "$ref": "#/$defs/types/count_packets"
+ },
+ "time": {
+ "$ref": "#/$defs/types/time_quantity"
+ }
+ },
+ "required": [
+ "packet",
+ "time"
+ ]
+ },
"value_list_with_unit_and_stats": {
"description": "Reusable composite type, multiple values together with their unit of measurement and derived statistics.",
"type": "object",
}
},
"avg": {
- "description": "A derived quantity, present only in info output files. It is the arithmetic average of the values list.",
+ "description": "A derived quantity. It is the arithmetic average of the values list.",
"$ref": "#/$defs/types/nonnegative_number"
},
"stdev": {
- "description": "A derived quantity, present only in info output files. It is the standard deviation for the values list, as computed by jumpavg library.",
+ "description": "A derived quantity. It is the standard deviation for the values list, as computed by jumpavg library.",
"$ref": "#/$defs/types/nonnegative_number"
},
"unit": {
]
},
"bandwidth_list": {
- "description": "Reusable composite type, multiple bandwidth values. This is a derived entity, thus it only appears in info output, and only if rate unit is pps.",
+ "description": "Reusable composite type, multiple bandwidth values. This is a derived quantity.",
"allOf": [
{
"$ref": "#/$defs/types/value_list_with_unit_and_stats"
---
-$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.1.0
+$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.2.0
$schema: https://json-schema.org/draft/2020-12/schema
description: >-
- Schema for info output of test case.
+ Schema for output of test case.
allOf:
- description: >-
The main structure, without conditional relations between fields yet.
properties:
duration:
description: >-
- A derived quantity, present only in info output files.
- Difference between start_time and end_time, in seconds.
+ A derived quantity. Difference between start_time and end_time,
+ in seconds.
$ref: "#/$defs/types/nonnegative_number"
dut_type:
description: >-
required:
- type
- critical_rate
+ - description: >-
+ Result type RECONF case.
+ additionalProperties: false
+ properties:
+ type:
+ const: reconf
+ aggregate_rate:
+ description: >-
+ Load used when reconfiguring, found as NDR lower
+ bound. This is an aggregate (bidirectional)
+ rate. Note that packets which the Traffic
+ Generator did not send, are also counted as lost
+ packets.
+ $ref: "#/$defs/types/rate_with_bandwidth"
+ loss:
+ description: >-
+ Number of packets lost during reconfiguration,
+ with the time that equals packet loss divided by
+ packet rate.
+ $ref: "#/$defs/types/packet_with_time"
+ required:
+ - loss
+ - aggregate_rate
start_time:
description: >-
UTC date and time in RFC 3339 format, specifying calendar time
type: string
test_id:
description: >-
- A derived quantity, present only in info output files.
+ A derived quantity.
It is the most complete and unique identifier for a test case.
This property has a value, of the following form:
{suite_name}.{test_name}
minLength: 3
test_name_long:
description: >-
- A derived quantity, present only in info output files.
+ A derived quantity.
This property has a value, of the following form:
{nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part}
Here, suite part is very similar to suite tag,
minLength: 3
test_name_short:
description: >-
- A derived quantity, present only in info output files.
+ A derived quantity.
This property has a value very similar to suite tag,
but additionally may contain a prefix describing NIC driver used
(if it is not the default one, drv_vfio_pci for VPP tests).
minLength: 3
test_type:
description: >-
- A derived quantity, present only in info output files.
+ A derived quantity.
Test type identifier, PAL uses it to group similar tests,
e.g. for comparison tables.
Ideally, this information should be parseable from test name,
CSIT model version (semver format)
the exporting code adhered to.
type: string
- const: 1.1.0
+ const: 1.2.0
required:
- duration
- dut_type
may allow more units.
enum:
- bps
+ count_packets:
+ description: >-
+ Type, for counting packets.
+ allOf:
+ - $ref: "#/$defs/types/value_with_unit"
+ - properties:
+ value:
+ description: >-
+ A number of packets of interest.
+ unit:
+ description: >-
+ Unit suitable for displaying packet counts.
+ enum:
+ - packets
+ time_quantity:
+ description: >-
+ Reusable type, for various time quantites.
+ allOf:
+ - $ref: "#/$defs/types/value_with_unit"
+ - properties:
+ value:
+ description: >-
+ Unless specified otherwise, this is a duration
+ between two events.
+ unit:
+ description: >-
+ Only seconds are the unit supported for time
+ quantities.
+ enum:
+ - s
value_with_unit:
description: >-
Reusable composite type, value together with its
$ref: "#/$defs/types/bandwidth"
required:
- rate
+ packet_with_time:
+ description: >-
+ Reusable composite type, joining packet count with the
+ time quantity.
+ type: object
+ additionalProperties: false
+ properties:
+ packet:
+ $ref: "#/$defs/types/count_packets"
+ time:
+ $ref: "#/$defs/types/time_quantity"
+ required:
+ - packet
+ - time
value_list_with_unit_and_stats:
description: >-
Reusable composite type, multiple values together with their
minItmes: 1
items:
description: >-
- Numeric value, context specified elsewhere.
- The only assumption is that the value is nonnegative.
+ Numeric value, context specified elsewhere. The only
+ assumption is that the value is nonnegative.
$ref: "#/$defs/types/nonnegative_number"
avg:
description: >-
- A derived quantity, present only in info output files.
- It is the arithmetic average of the values list.
+ A derived quantity. It is the arithmetic average of the
+ values list.
$ref: "#/$defs/types/nonnegative_number"
stdev:
description: >-
- A derived quantity, present only in info output files.
- It is the standard deviation for the values list,
- as computed by jumpavg library.
+ A derived quantity. It is the standard deviation for the
+ values list, as computed by jumpavg library.
$ref: "#/$defs/types/nonnegative_number"
unit:
description: >-
$ref: "#/$defs/types/rate_unit"
bandwidth_list:
description: >-
- Reusable composite type, multiple bandwidth values.
- This is a derived entity, thus it only appears in info output,
- and only if rate unit is pps.
+ Reusable composite type, multiple bandwidth values. This is a
+ derived quantity.
allOf:
- $ref: "#/$defs/types/value_list_with_unit_and_stats"
- properties:
Version
~~~~~~~
-This document is valid for CSIT model version 1.1.0.
+This document is valid for CSIT model version 1.2.0.
It is recommended to use semantic versioning: https://semver.org/
That means, if the new model misses a field present in the old model,
"""Constants used in CSIT."""
# Version for CSIT data model. See docs/model/.
- MODEL_VERSION = u"1.1.0"
+ MODEL_VERSION = u"1.2.0"
# Global off-switch in case JSON export is large or slow.
EXPORT_JSON = get_optimistic_bool_from_env(u"EXPORT_JSON")
upper_or_lower = u"upper" if u"upper" in text else u"lower"
ndr_or_pdr = u"ndr" if u"ndr" in text else u"pdr"
- data = get_export_data()
- result_node = data[u"result"]
+ result_node = get_export_data()[u"result"]
result_node[u"type"] = result_type
rate_item = dict(rate=dict(value=value, unit=unit))
if bandwidth:
:type text: str
:type latency: 1-tuple or 2-tuple of str
"""
- data = get_export_data()
- result_node = data[u"result"]
+ result_node = get_export_data()[u"result"]
percent = 0
if u"90" in text:
percent = 90
return
_add_latency(result_node, percent, u"reverse", latency[1])
+
+def export_reconf_result(packet_rate, packet_loss, bandwidth):
+ """Export the results from a reconf test.
+
+ :param packet_rate: Aggregate offered load in packets per second.
+ :param packet_loss: How many of the packets were dropped or unsent.
+ :param bandwidth: The offered load recomputed into L1 bits per second.
+ :type packet_rate: float
+ :type packet_loss: int
+ :type bandwidth: float
+ """
+ result_node = get_export_data()["result"]
+ result_node["type"] = "reconf"
+
+ time_loss = int(packet_loss) / float(packet_rate)
+ result_node["aggregate_rate"] = dict(
+ bandwidth=dict(
+ unit="bps",
+ value=float(bandwidth)
+ ),
+ rate=dict(
+ unit="pps",
+ value=float(packet_rate)
+ )
+ )
+ result_node["loss"] = dict(
+ packet=dict(
+ unit="packets",
+ value=int(packet_loss)
+ ),
+ time=dict(
+ unit="s",
+ value=time_loss
+ )
+ )
+
+
def append_telemetry(telemetry_item):
"""Append telemetry entry to proper place so it is dumped into json.
| |
| | [Arguments] | ${result}
| |
-| | ${ppta} = | Get Packets Per Transaction Aggregated
-| | ${packet_rate} = | Evaluate | ${result.target_tr} * ${ppta}
+| | ${bandwidth} | ${packet_rate}= | Compute Bandwidth | ${result.target_tr}
| | ${packet_loss} = | Set Variable | ${result.loss_count}
| | ${time_loss} = | Evaluate | ${packet_loss} / ${packet_rate}
| | Set Test Message | Packets lost due to reconfig: ${packet_loss}
| | Set Test Message | ${\n}Implied time lost: ${time_loss} | append=yes
+| | Export Reconf Result | ${packet_rate} | ${packet_loss} | ${bandwidth * 1e9}
| Display result of NDRPDR search
| | [Documentation]
Code Review / csit.git / commitdiff