-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2023 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
---
-$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.0.0
+$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.3.0
$schema: https://json-schema.org/draft/2020-12/schema
description: >-
- Schema for info output of test case.
+ Schema for output of test case.
allOf:
- description: >-
The main structure, without conditional relations between fields yet.
properties:
duration:
description: >-
- A derived quantity, present only in info output files.
- Difference between start_time and end_time, in seconds.
+ A derived quantity. Difference between start_time and end_time,
+ in seconds.
$ref: "#/$defs/types/nonnegative_number"
dut_type:
description: >-
DUT type used, e.g. VPP or DPDK.
type: string
+ minLength: 1
dut_version:
description: >-
Version string appropriate to DUT type used.
type: string
+ minLength: 1
end_time:
description: >-
UTC date and time in RFC 3339 format, specifying calendar time
description: >-
Host identifier, usually numeric IPv4 address.
type: string
- log:
+ telemetry:
description: >-
- No log items are implemented in the current version,
- but the (empty) list is present to simplify logic
- in multi-version importers.
- $ref: "#/$defs/types/empty_array"
+ Array of telemetry entries. Each entry represent one captured
+ metric.
+ type: array
+ minItems: 0
+ items:
+ description: >-
+ Telemetry entry.
+ type: string
message:
description: >-
If passed is true, this value is empty.
required:
- type
- critical_rate
+ - description: >-
+ Result type RECONF case.
+ additionalProperties: false
+ properties:
+ type:
+ const: reconf
+ aggregate_rate:
+ description: >-
+ Load used when reconfiguring, found as NDR lower
+ bound. This is an aggregate (bidirectional)
+ rate. Note that packets which the Traffic
+ Generator did not send, are also counted as lost
+ packets.
+ $ref: "#/$defs/types/rate_with_bandwidth"
+ loss:
+ description: >-
+ Number of packets lost during reconfiguration,
+ with the time that equals packet loss divided by
+ packet rate.
+ $ref: "#/$defs/types/packet_with_time"
+ required:
+ - type
+ - loss
+ - aggregate_rate
+ - description: >-
+ Result type HOSTSTACK case.
+ additionalProperties: false
+ properties:
+ type:
+ const: hoststack
+ bandwidth:
+ description: >-
+ Goodput measured in bits per second.
+ $ref: "#/$defs/types/bandwidth"
+ completed_requests:
+ description: >-
+ Number of completed requests.
+ $ref: "#/$defs/types/count_requests"
+ failed_requests:
+ description: >-
+ Number of failed requests.
+ $ref: "#/$defs/types/count_requests"
+ retransmits:
+ description: >-
+ Number of retransmits.
+ $ref: "#/$defs/types/count_packets"
+ latency:
+ description: >-
+ Value and unit of latency.
+ $ref: "#/$defs/types/value_with_unit"
+ duration:
+ description: >-
+ The relative time difference (in seconds)
+ between program start and end.
+ $ref: "#/$defs/types/time_quantity"
+ rate:
+ description: >-
+ RPS or CPS rate, with corresponding unit, as
+ reported by TG.
+ $ref: "#/$defs/types/rate_without_bandwidth"
+ required:
+ - type
+ - bandwidth
start_time:
description: >-
UTC date and time in RFC 3339 format, specifying calendar time
type: string
test_id:
description: >-
- A derived quantity, present only in info output files.
+ A derived quantity.
It is the most complete and unique identifier for a test case.
This property has a value, of the following form:
{suite_name}.{test_name}
minLength: 3
test_name_long:
description: >-
- A derived quantity, present only in info output files.
+ A derived quantity.
This property has a value, of the following form:
{nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part}
Here, suite part is very similar to suite tag,
minLength: 3
test_name_short:
description: >-
- A derived quantity, present only in info output files.
+ A derived quantity.
This property has a value very similar to suite tag,
but additionally may contain a prefix describing NIC driver used
(if it is not the default one, drv_vfio_pci for VPP tests).
minLength: 3
test_type:
description: >-
- A derived quantity, present only in info output files.
+ A derived quantity.
Test type identifier, PAL uses it to group similar tests,
e.g. for comparison tables.
Ideally, this information should be parseable from test name,
- ndrpdr
- reconf
- soak
- - vsap
+ tg_type:
+ description: >-
+ TG type used, e.g. TREX.
+ type: string
+ minLength: 1
+ tg_version:
+ description: >-
+ Version string appropriate to TG type used.
+ type: string
+ minLength: 1
version:
description: >-
CSIT model version (semver format)
the exporting code adhered to.
type: string
- const: 1.0.0
+ const: 1.3.0
required:
- duration
- dut_type
- dut_version
- end_time
- hosts
- - log
+ - telemetry
- message
- passed
- result
- test_name_long
- test_name_short
- test_type
+ - tg_type
+ - tg_version
- version
- description: >-
Subschema validating relation between status and message.
const: false
message:
minLength: 1
-- description: >-
- Subschema validating relation between dut_type and dut_version.
- oneOf:
- - description: >-
- Subschema for tests with no DUT, e.g. TRex self-test.
- type: object
- properties:
- dut_type:
- const: none
- dut_version:
- const: ""
- - description: >-
- Subschema for DUT type VPP.
- type: object
- properties:
- dut_type:
- const: VPP
- dut_version:
- minLength: 1
- - description: >-
- Subschema for DUT type DPDK.
- type: object
- properties:
- dut_type:
- const: DPDK
- dut_version:
- minLength: 1
$defs:
types:
maxItems: 0
rate_unit:
description: >-
- Packets per second (pps) or connections per second (cps).
+ Packets per second (pps),
+ connections per second (cps),
+ requests per second (rps).
type: string
enum:
- pps
- cps
+ - rps
bandwidth_unit:
description: >-
Unit of measurement for bandwidth values.
may allow more units.
enum:
- bps
+ count_packets:
+ description: >-
+ Type, for counting packets.
+ allOf:
+ - $ref: "#/$defs/types/value_with_unit"
+ - properties:
+ value:
+ description: >-
+ A number of packets of interest.
+ unit:
+ description: >-
+ Unit suitable for displaying packet counts.
+ enum:
+ - packets
+ count_requests:
+ description: >-
+ Type, for counting requests.
+ allOf:
+ - $ref: "#/$defs/types/value_with_unit"
+ - properties:
+ value:
+ description: >-
+ A number of requests of interest.
+ unit:
+ description: >-
+ Unit suitable for displaying request counts.
+ enum:
+ - requests
+ time_quantity:
+ description: >-
+ Reusable type, for various time quantites.
+ allOf:
+ - $ref: "#/$defs/types/value_with_unit"
+ - properties:
+ value:
+ description: >-
+ Unless specified otherwise, this is a duration
+ between two events.
+ unit:
+ description: >-
+ Only seconds are the unit supported for time
+ quantities.
+ enum:
+ - s
value_with_unit:
description: >-
Reusable composite type, value together with its
- properties:
value:
description: >-
- Bandwidth value computed
- from the corresponding rate.
+ Bandwidth value computed from the corresponding
+ rate.
unit:
$ref: "#/$defs/types/bandwidth_unit"
rate_with_bandwidth:
$ref: "#/$defs/types/bandwidth"
required:
- rate
+ packet_with_time:
+ description: >-
+ Reusable composite type, joining packet count with the
+ time quantity.
+ type: object
+ additionalProperties: false
+ properties:
+ packet:
+ $ref: "#/$defs/types/count_packets"
+ time:
+ $ref: "#/$defs/types/time_quantity"
+ required:
+ - packet
+ - time
value_list_with_unit_and_stats:
description: >-
Reusable composite type, multiple values together with their
minItmes: 1
items:
description: >-
- Numeric value, context specified elsewhere.
- The only assumption is that the value is nonnegative.
+ Numeric value, context specified elsewhere. The only
+ assumption is that the value is nonnegative.
$ref: "#/$defs/types/nonnegative_number"
avg:
description: >-
- A derived quantity, present only in info output files.
- It is the arithmetic average of the values list.
+ A derived quantity. It is the arithmetic average of the
+ values list.
$ref: "#/$defs/types/nonnegative_number"
stdev:
description: >-
- A derived quantity, present only in info output files.
- It is the standard deviation for the values list,
- as computed by jumpavg library.
+ A derived quantity. It is the standard deviation for the
+ values list, as computed by jumpavg library.
$ref: "#/$defs/types/nonnegative_number"
unit:
description: >-
$ref: "#/$defs/types/rate_unit"
bandwidth_list:
description: >-
- Reusable composite type, multiple bandwidth values.
- This is a derived entity, thus it only appears in info output,
- and only if rate unit is pps.
+ Reusable composite type, multiple bandwidth values. This is a
+ derived quantity.
allOf:
- $ref: "#/$defs/types/value_list_with_unit_and_stats"
- properties:
Code Review / csit.git / blobdiff