From 420226fcfd8cab460f632c1401d6012dc353f6fd Mon Sep 17 00:00:00 2001 From: pmikus Date: Tue, 6 Dec 2022 12:54:25 +0000 Subject: [PATCH] feat(model): Reconf type Signed-off-by: pmikus Change-Id: Ibfab25ee82eaa207987f4070cf2386ea9d0781cd --- .../current/schema/test_case.info.schema.json | 102 +++++++++++++++++--- .../current/schema/test_case.info.schema.yaml | 103 +++++++++++++++++---- docs/model/current/top.rst | 2 +- resources/libraries/python/Constants.py | 2 +- resources/libraries/python/model/ExportResult.py | 43 ++++++++- .../robot/performance/performance_display.robot | 4 +- 6 files changed, 218 insertions(+), 38 deletions(-) diff --git a/docs/model/current/schema/test_case.info.schema.json b/docs/model/current/schema/test_case.info.schema.json index 6aed4d255b..7bdfa27f8d 100644 --- a/docs/model/current/schema/test_case.info.schema.json +++ b/docs/model/current/schema/test_case.info.schema.json @@ -1,7 +1,7 @@ { - "$id": "https://fd.io/FIXME/CSIT/UTI/test_case/info/1.1.0", + "$id": "https://fd.io/FIXME/CSIT/UTI/test_case/info/1.2.0", "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Schema for info output of test case.", + "description": "Schema for output of test case.", "allOf": [ { "description": "The main structure, without conditional relations between fields yet.", @@ -9,7 +9,7 @@ "additionalProperties": false, "properties": { "duration": { - "description": "A derived quantity, present only in info output files. Difference between start_time and end_time, in seconds.", + "description": "A derived quantity. Difference between start_time and end_time, in seconds.", "$ref": "#/$defs/types/nonnegative_number" }, "dut_type": { @@ -141,6 +141,27 @@ "type", "critical_rate" ] + }, + { + "description": "Result type RECONF case.", + "additionalProperties": false, + "properties": { + "type": { + "const": "reconf" + }, + "aggregate_rate": { + "description": "Load used when reconfiguring, found as NDR lower bound. This is an aggregate (bidirectional) rate. Note that packets which the Traffic Generator did not send, are also counted as lost packets.", + "$ref": "#/$defs/types/rate_with_bandwidth" + }, + "loss": { + "description": "Number of packets lost during reconfiguration, with the time that equals packet loss divided by packet rate.", + "$ref": "#/$defs/types/packet_with_time" + } + }, + "required": [ + "loss", + "aggregate_rate" + ] } ] } @@ -162,22 +183,22 @@ "type": "string" }, "test_id": { - "description": "A derived quantity, present only in info output files. It is the most complete and unique identifier for a test case. This property has a value, of the following form: {suite_name}.{test_name} Here, suite name comes from SUITE_NAME robot variable, test name comes from TEST_NAME robot variable, but both are converted to lower case, and spaces are replaced by underscores.", + "description": "A derived quantity. It is the most complete and unique identifier for a test case. This property has a value, of the following form: {suite_name}.{test_name} Here, suite name comes from SUITE_NAME robot variable, test name comes from TEST_NAME robot variable, but both are converted to lower case, and spaces are replaced by underscores.", "type": "string", "minLength": 3 }, "test_name_long": { - "description": "A derived quantity, present only in info output files. This property has a value, of the following form: {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part} Here, suite part is very similar to suite tag, but additionally may contain a prefix describing NIC driver used (if it is not the default one, drv_vfio_pci for VPP tests). Any space is replaced by underscore and letters are lower case.", + "description": "A derived quantity. This property has a value, of the following form: {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part} Here, suite part is very similar to suite tag, but additionally may contain a prefix describing NIC driver used (if it is not the default one, drv_vfio_pci for VPP tests). Any space is replaced by underscore and letters are lower case.", "type": "string", "minLength": 3 }, "test_name_short": { - "description": "A derived quantity, present only in info output files. This property has a value very similar to suite tag, but additionally may contain a prefix describing NIC driver used (if it is not the default one, drv_vfio_pci for VPP tests). Any space is replaced by underscore and letters are lower case.", + "description": "A derived quantity. This property has a value very similar to suite tag, but additionally may contain a prefix describing NIC driver used (if it is not the default one, drv_vfio_pci for VPP tests). Any space is replaced by underscore and letters are lower case.", "type": "string", "minLength": 3 }, "test_type": { - "description": "A derived quantity, present only in info output files. Test type identifier, PAL uses it to group similar tests, e.g. for comparison tables. Ideally, this information should be parseable from test name, but the current naming scheme is not simple/consistent enough. The current implementation queries the robot test tags. The resulting value is frequently identical to result type, but this schema version does not require any relation there, as PAL may want to group tests differently.", + "description": "A derived quantity. Test type identifier, PAL uses it to group similar tests, e.g. for comparison tables. Ideally, this information should be parseable from test name, but the current naming scheme is not simple/consistent enough. The current implementation queries the robot test tags. The resulting value is frequently identical to result type, but this schema version does not require any relation there, as PAL may want to group tests differently.", "type": "string", "enum": [ "device", @@ -203,7 +224,7 @@ "version": { "description": "CSIT model version (semver format) the exporting code adhered to.", "type": "string", - "const": "1.1.0" + "const": "1.2.0" } }, "required": [ @@ -298,6 +319,48 @@ "bps" ] }, + "count_packets": { + "description": "Type, for counting packets.", + "allOf": [ + { + "$ref": "#/$defs/types/value_with_unit" + }, + { + "properties": { + "value": { + "description": "A number of packets of interest." + }, + "unit": { + "description": "Unit suitable for displaying packet counts.", + "enum": [ + "packets" + ] + } + } + } + ] + }, + "time_quantity": { + "description": "Reusable type, for various time quantites.", + "allOf": [ + { + "$ref": "#/$defs/types/value_with_unit" + }, + { + "properties": { + "value": { + "description": "Unless specified otherwise, this is a duration between two events." + }, + "unit": { + "description": "Only seconds are the unit supported for time quantities.", + "enum": [ + "s" + ] + } + } + } + ] + }, "value_with_unit": { "description": "Reusable composite type, value together with its unit of measurement.", "type": "object", @@ -370,6 +433,23 @@ "rate" ] }, + "packet_with_time": { + "description": "Reusable composite type, joining packet count with the time quantity.", + "type": "object", + "additionalProperties": false, + "properties": { + "packet": { + "$ref": "#/$defs/types/count_packets" + }, + "time": { + "$ref": "#/$defs/types/time_quantity" + } + }, + "required": [ + "packet", + "time" + ] + }, "value_list_with_unit_and_stats": { "description": "Reusable composite type, multiple values together with their unit of measurement and derived statistics.", "type": "object", @@ -385,11 +465,11 @@ } }, "avg": { - "description": "A derived quantity, present only in info output files. It is the arithmetic average of the values list.", + "description": "A derived quantity. It is the arithmetic average of the values list.", "$ref": "#/$defs/types/nonnegative_number" }, "stdev": { - "description": "A derived quantity, present only in info output files. It is the standard deviation for the values list, as computed by jumpavg library.", + "description": "A derived quantity. It is the standard deviation for the values list, as computed by jumpavg library.", "$ref": "#/$defs/types/nonnegative_number" }, "unit": { @@ -425,7 +505,7 @@ ] }, "bandwidth_list": { - "description": "Reusable composite type, multiple bandwidth values. This is a derived entity, thus it only appears in info output, and only if rate unit is pps.", + "description": "Reusable composite type, multiple bandwidth values. This is a derived quantity.", "allOf": [ { "$ref": "#/$defs/types/value_list_with_unit_and_stats" diff --git a/docs/model/current/schema/test_case.info.schema.yaml b/docs/model/current/schema/test_case.info.schema.yaml index 9f78a41d9e..022061aa39 100644 --- a/docs/model/current/schema/test_case.info.schema.yaml +++ b/docs/model/current/schema/test_case.info.schema.yaml @@ -13,10 +13,10 @@ --- -$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.1.0 +$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.2.0 $schema: https://json-schema.org/draft/2020-12/schema description: >- - Schema for info output of test case. + Schema for output of test case. allOf: - description: >- The main structure, without conditional relations between fields yet. @@ -25,8 +25,8 @@ allOf: properties: duration: description: >- - A derived quantity, present only in info output files. - Difference between start_time and end_time, in seconds. + A derived quantity. Difference between start_time and end_time, + in seconds. $ref: "#/$defs/types/nonnegative_number" dut_type: description: >- @@ -213,6 +213,29 @@ allOf: required: - type - critical_rate + - description: >- + Result type RECONF case. + additionalProperties: false + properties: + type: + const: reconf + aggregate_rate: + description: >- + Load used when reconfiguring, found as NDR lower + bound. This is an aggregate (bidirectional) + rate. Note that packets which the Traffic + Generator did not send, are also counted as lost + packets. + $ref: "#/$defs/types/rate_with_bandwidth" + loss: + description: >- + Number of packets lost during reconfiguration, + with the time that equals packet loss divided by + packet rate. + $ref: "#/$defs/types/packet_with_time" + required: + - loss + - aggregate_rate start_time: description: >- UTC date and time in RFC 3339 format, specifying calendar time @@ -236,7 +259,7 @@ allOf: type: string test_id: description: >- - A derived quantity, present only in info output files. + A derived quantity. It is the most complete and unique identifier for a test case. This property has a value, of the following form: {suite_name}.{test_name} @@ -248,7 +271,7 @@ allOf: minLength: 3 test_name_long: description: >- - A derived quantity, present only in info output files. + A derived quantity. This property has a value, of the following form: {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part} Here, suite part is very similar to suite tag, @@ -259,7 +282,7 @@ allOf: minLength: 3 test_name_short: description: >- - A derived quantity, present only in info output files. + A derived quantity. This property has a value very similar to suite tag, but additionally may contain a prefix describing NIC driver used (if it is not the default one, drv_vfio_pci for VPP tests). @@ -268,7 +291,7 @@ allOf: minLength: 3 test_type: description: >- - A derived quantity, present only in info output files. + A derived quantity. Test type identifier, PAL uses it to group similar tests, e.g. for comparison tables. Ideally, this information should be parseable from test name, @@ -302,7 +325,7 @@ allOf: CSIT model version (semver format) the exporting code adhered to. type: string - const: 1.1.0 + const: 1.2.0 required: - duration - dut_type @@ -377,6 +400,36 @@ $defs: may allow more units. enum: - bps + count_packets: + description: >- + Type, for counting packets. + allOf: + - $ref: "#/$defs/types/value_with_unit" + - properties: + value: + description: >- + A number of packets of interest. + unit: + description: >- + Unit suitable for displaying packet counts. + enum: + - packets + time_quantity: + description: >- + Reusable type, for various time quantites. + allOf: + - $ref: "#/$defs/types/value_with_unit" + - properties: + value: + description: >- + Unless specified otherwise, this is a duration + between two events. + unit: + description: >- + Only seconds are the unit supported for time + quantities. + enum: + - s value_with_unit: description: >- Reusable composite type, value together with its @@ -443,6 +496,20 @@ $defs: $ref: "#/$defs/types/bandwidth" required: - rate + packet_with_time: + description: >- + Reusable composite type, joining packet count with the + time quantity. + type: object + additionalProperties: false + properties: + packet: + $ref: "#/$defs/types/count_packets" + time: + $ref: "#/$defs/types/time_quantity" + required: + - packet + - time value_list_with_unit_and_stats: description: >- Reusable composite type, multiple values together with their @@ -457,19 +524,18 @@ $defs: minItmes: 1 items: description: >- - Numeric value, context specified elsewhere. - The only assumption is that the value is nonnegative. + Numeric value, context specified elsewhere. The only + assumption is that the value is nonnegative. $ref: "#/$defs/types/nonnegative_number" avg: description: >- - A derived quantity, present only in info output files. - It is the arithmetic average of the values list. + A derived quantity. It is the arithmetic average of the + values list. $ref: "#/$defs/types/nonnegative_number" stdev: description: >- - A derived quantity, present only in info output files. - It is the standard deviation for the values list, - as computed by jumpavg library. + A derived quantity. It is the standard deviation for the + values list, as computed by jumpavg library. $ref: "#/$defs/types/nonnegative_number" unit: description: >- @@ -501,9 +567,8 @@ $defs: $ref: "#/$defs/types/rate_unit" bandwidth_list: description: >- - Reusable composite type, multiple bandwidth values. - This is a derived entity, thus it only appears in info output, - and only if rate unit is pps. + Reusable composite type, multiple bandwidth values. This is a + derived quantity. allOf: - $ref: "#/$defs/types/value_list_with_unit_and_stats" - properties: diff --git a/docs/model/current/top.rst b/docs/model/current/top.rst index e824225698..3f7f976adf 100644 --- a/docs/model/current/top.rst +++ b/docs/model/current/top.rst @@ -22,7 +22,7 @@ especially the export side (UTI), not import side (PAL). Version ~~~~~~~ -This document is valid for CSIT model version 1.1.0. +This document is valid for CSIT model version 1.2.0. It is recommended to use semantic versioning: https://semver.org/ That means, if the new model misses a field present in the old model, diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py index ae1a64d2d2..fa18a5b4a2 100644 --- a/resources/libraries/python/Constants.py +++ b/resources/libraries/python/Constants.py @@ -120,7 +120,7 @@ class Constants: """Constants used in CSIT.""" # Version for CSIT data model. See docs/model/. - MODEL_VERSION = u"1.1.0" + MODEL_VERSION = u"1.2.0" # Global off-switch in case JSON export is large or slow. EXPORT_JSON = get_optimistic_bool_from_env(u"EXPORT_JSON") diff --git a/resources/libraries/python/model/ExportResult.py b/resources/libraries/python/model/ExportResult.py index 31840c9deb..b5de27ebe5 100644 --- a/resources/libraries/python/model/ExportResult.py +++ b/resources/libraries/python/model/ExportResult.py @@ -143,8 +143,7 @@ def export_search_bound(text, value, unit, bandwidth=None): upper_or_lower = u"upper" if u"upper" in text else u"lower" ndr_or_pdr = u"ndr" if u"ndr" in text else u"pdr" - data = get_export_data() - result_node = data[u"result"] + result_node = get_export_data()[u"result"] result_node[u"type"] = result_type rate_item = dict(rate=dict(value=value, unit=unit)) if bandwidth: @@ -196,8 +195,7 @@ def export_ndrpdr_latency(text, latency): :type text: str :type latency: 1-tuple or 2-tuple of str """ - data = get_export_data() - result_node = data[u"result"] + result_node = get_export_data()[u"result"] percent = 0 if u"90" in text: percent = 90 @@ -211,6 +209,43 @@ def export_ndrpdr_latency(text, latency): return _add_latency(result_node, percent, u"reverse", latency[1]) + +def export_reconf_result(packet_rate, packet_loss, bandwidth): + """Export the results from a reconf test. + + :param packet_rate: Aggregate offered load in packets per second. + :param packet_loss: How many of the packets were dropped or unsent. + :param bandwidth: The offered load recomputed into L1 bits per second. + :type packet_rate: float + :type packet_loss: int + :type bandwidth: float + """ + result_node = get_export_data()["result"] + result_node["type"] = "reconf" + + time_loss = int(packet_loss) / float(packet_rate) + result_node["aggregate_rate"] = dict( + bandwidth=dict( + unit="bps", + value=float(bandwidth) + ), + rate=dict( + unit="pps", + value=float(packet_rate) + ) + ) + result_node["loss"] = dict( + packet=dict( + unit="packets", + value=int(packet_loss) + ), + time=dict( + unit="s", + value=time_loss + ) + ) + + def append_telemetry(telemetry_item): """Append telemetry entry to proper place so it is dumped into json. diff --git a/resources/libraries/robot/performance/performance_display.robot b/resources/libraries/robot/performance/performance_display.robot index 1313788d5e..d230bcb35a 100644 --- a/resources/libraries/robot/performance/performance_display.robot +++ b/resources/libraries/robot/performance/performance_display.robot @@ -86,12 +86,12 @@ | | | | [Arguments] | ${result} | | -| | ${ppta} = | Get Packets Per Transaction Aggregated -| | ${packet_rate} = | Evaluate | ${result.target_tr} * ${ppta} +| | ${bandwidth} | ${packet_rate}= | Compute Bandwidth | ${result.target_tr} | | ${packet_loss} = | Set Variable | ${result.loss_count} | | ${time_loss} = | Evaluate | ${packet_loss} / ${packet_rate} | | Set Test Message | Packets lost due to reconfig: ${packet_loss} | | Set Test Message | ${\n}Implied time lost: ${time_loss} | append=yes +| | Export Reconf Result | ${packet_rate} | ${packet_loss} | ${bandwidth * 1e9} | Display result of NDRPDR search | | [Documentation] -- 2.16.6