resources/model_schema/test_case.schema.yaml

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 ---
  15
  16 $id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.5.1
  17 $schema: https://json-schema.org/draft/2020-12/schema
  18 description: >-
  19     Schema for output of test case.
  20 allOf:
  21 -   description: >-
  22         The main structure, without conditional relations between fields yet.
  23     type: object
  24     additionalProperties: false
  25     properties:
  26         duration:
  27             description: >-
  28                 A derived quantity. Difference between start_time and end_time,
  29                 in seconds.
  30             $ref: "#/$defs/types/nonnegative_number"
  31         dut_type:
  32             description: >-
  33                 DUT type used, e.g. VPP or DPDK.
  34             type: string
  35             minLength: 1
  36         dut_version:
  37             description: >-
  38                 Version string appropriate to DUT type used.
  39             type: string
  40             minLength: 1
  41         end_time:
  42             description: >-
  43                 UTC date and time in RFC 3339 format, specifying calendar time
  44                 just before test case ended (at the end of test case teardown).
  45             $ref: "#/$defs/types/date_time"
  46         hosts:
  47             description: >-
  48                 Array of hosts this test interacted with.
  49                 This can be used for identifying testbed number.
  50                 Valid tests shoud interact with at least one DUT or TG.
  51                 The array is usually sorted, but that is not a requirement.
  52             type: array
  53             minItems: 1
  54             items:
  55                 description: >-
  56                     Host identifier, usually numeric IPv4 address.
  57                 type: string
  58         telemetry:
  59             description: >-
  60                 Array of telemetry entries. Each entry represent one captured
  61                 metric.
  62             type: array
  63             minItems: 0
  64             items:
  65                 description: >-
  66                     Telemetry entry.
  67                 type: string
  68         message:
  69             description: >-
  70                 If passed is true, this value is empty.
  71                 Otherwise, value taken directly from TEST_MESSAGE
  72                 Robot variable, read at the end of test case
  73                 (in test teardown, before export and validation).
  74                 It contains information from the exception
  75                 that caused the failure, probably with additional
  76                 exceptions from teardown keywords.
  77             type: string
  78         passed:
  79             description: >-
  80                 Value set accordingly to TEST_STATUS Robot variable,
  81                 true if and only if the status is "PASS".
  82                 The status is read at the end of test case
  83                 (in test teardown, before export and validation).
  84             type: boolean
  85         result:
  86             type: object
  87             allOf:
  88             -   description: >-
  89                     Sub-schema common for all cases,
  90                     only result type identifier defined here.
  91                 properties:
  92                     type:
  93                         description: >-
  94                             Identifier of which result type case is applied.
  95                         type: string
  96                 required:
  97                 -   type
  98             -   oneOf:
  99                 -   description: >-
 100                         Result type for unknown case.
 101                         This case represents a test with no specific result
 102                         (outside message), e.g. device test;
 103                         or a test with result not parsed into
 104                         this version of model yet, e.g. GSO test.
 105                     additionalProperties: false
 106                     properties:
 107                         type:
 108                             const: unknown
 109                 -   description: >-
 110                         Result type MRR case.
 111                     additionalProperties: false
 112                     properties:
 113                         type:
 114                             const: mrr
 115                         receive_rate:
 116                             description: >-
 117                                 The results refer to receive rates for multiple
 118                                 MRR trials. For PPS, these are aggregate
 119                                 (bidirectional) rates.
 120                                 Currently, the tests are exporting
 121                                 approximated receive rates.
 122                                 That means the actual trial duration
 123                                 is measured (as opposed to trusting traffic
 124                                 generator to honor its target duration),
 125                                 so the resulting values contain noise
 126                                 from time measurement, and can be lower
 127                                 than the real performance
 128                                 (due to various time overheads).
 129                                 Bandwidth values are supported, but currently
 130                                 Robot does not export them.
 131                             $ref: "#/$defs/types/rate_list_with_bandwidth"
 132                     required:
 133                     -   type
 134                     -   receive_rate
 135                 -   description: >-
 136                         Result type NDRPDR case.
 137                     additionalProperties: false
 138                     properties:
 139                         type:
 140                             const: ndrpdr
 141                         ndr:
 142                             description: >-
 143                                 The results refer to search for NDR
 144                                 The accepted loss ratio for NDR is exact zero.
 145                                 Exceed ratio is 50%.
 146                                 For PPS, lowerbound is aggregate
 147                                 (bidirectional) conditional throughput
 148                                 (median forwarding rate among long trials),
 149                                 upperbound is missing.
 150                                 Note that packets the Traffic Generator
 151                                 did not send are also counted as lost packets.
 152                             $ref: "#/$defs/macros/lower_and_maybe_upper_rate"
 153                         pdr:
 154                             description: >-
 155                                 The results refer to search for PDR.
 156                                 The accepted loss ratio for PDR is 0.5%.
 157                                 Exceed ratio is 50%.
 158                                 For PPS, lowerbound is aggregate
 159                                 (bidirectional) conditional throughput
 160                                 (median forwarding rate among long trials),
 161                                 upperbound is missing.
 162                                 Note that packets the Traffic Generator
 163                                 did not send are also counted as lost packets.
 164                             $ref: "#/$defs/macros/lower_and_maybe_upper_rate"
 165                         latency_forward:
 166                             description: >-
 167                                 Object with results related to latency part
 168                                 of NDRPDR test, for forward traffic direction.
 169                                 It is the direction used in unidirectional
 170                                 traffic profiles.
 171                                 ASTF profiles and IMIX STL profiles
 172                                 do not support latency information,
 173                                 so for those tests this object is missing.
 174                                 It is also missing if Traffic Generator
 175                                 fails to return valid latency results
 176                                 for any other reasons,
 177                                 e.g. latency rate is too high for CPU/NIC used.
 178                             $ref: "#/$defs/macros/latency_for_loads"
 179                         latency_reverse:
 180                             description: >-
 181                                 Object with results related to latency part
 182                                 of NDRPDR test, for reverse traffic diration.
 183                                 This object is not present
 184                                 when unidirectional traffic profiles are used.
 185                                 ASTF profiles and IMIX STL profiles
 186                                 do not support latency information,
 187                                 so for those tests this object is missing.
 188                                 It is also missing if Traffic Generator
 189                                 fails to return valid latency results
 190                                 for any other reasons,
 191                                 e.g. latency rate is too high for CPU/NIC used.
 192                             $ref: "#/$defs/macros/latency_for_loads"
 193                     required:
 194                     -   type
 195                     -   ndr
 196                     -   pdr
 197                 -   description: >-
 198                         Result type SOAK case.
 199                     additionalProperties: false
 200                     properties:
 201                         type:
 202                             const: soak
 203                         critical_rate:
 204                             description: >-
 205                                 The results refer to bayesian estimate
 206                                 of critical rate corresponding to
 207                                 average loss ratio of 10^-7.
 208                                 For PPS, this is aggregate (bidirectional) rate.
 209                                 The bounds are computed from
 210                                 trial measurement results,
 211                                 but are not equal to any target load used.
 212                                 Note that packets the Traffic Generator
 213                                 did not send are also counted as lost packets.
 214                             $ref: "#/$defs/macros/lower_and_maybe_upper_rate"
 215                     required:
 216                     -   type
 217                     -   critical_rate
 218                 -   description: >-
 219                         Result type RECONF case.
 220                     additionalProperties: false
 221                     properties:
 222                         type:
 223                             const: reconf
 224                         aggregate_rate:
 225                             description: >-
 226                                 Load used when reconfiguring, found as NDR lower
 227                                 bound. This is an aggregate (bidirectional)
 228                                 rate. Note that packets which the Traffic
 229                                 Generator did not send, are also counted as lost
 230                                 packets.
 231                             $ref: "#/$defs/types/rate_with_bandwidth"
 232                         loss:
 233                             description: >-
 234                                 Number of packets lost during reconfiguration,
 235                                 with the time that equals packet loss divided by
 236                                 packet rate.
 237                             $ref: "#/$defs/types/packet_with_time"
 238                     required:
 239                     -   type
 240                     -   loss
 241                     -   aggregate_rate
 242                 -   description: >-
 243                         Result type HOSTSTACK case.
 244                     additionalProperties: false
 245                     properties:
 246                         type:
 247                             const: hoststack
 248                         bandwidth:
 249                             description: >-
 250                                 Goodput measured in bits per second.
 251                             $ref: "#/$defs/types/bandwidth"
 252                         completed_requests:
 253                             description: >-
 254                                 Number of completed requests.
 255                             $ref: "#/$defs/types/count_requests"
 256                         failed_requests:
 257                             description: >-
 258                                 Number of failed requests.
 259                             $ref: "#/$defs/types/count_requests"
 260                         retransmits:
 261                             description: >-
 262                                 Number of retransmits.
 263                             $ref: "#/$defs/types/count_packets"
 264                         latency:
 265                             description: >-
 266                                 Value and unit of latency.
 267                             $ref: "#/$defs/types/value_with_unit"
 268                         duration:
 269                             description: >-
 270                                 The relative time difference (in seconds)
 271                                 between program start and end.
 272                             $ref: "#/$defs/types/time_quantity"
 273                         rate:
 274                             description: >-
 275                                 RPS or CPS rate, with corresponding unit, as
 276                                 reported by TG.
 277                             $ref: "#/$defs/types/rate_without_bandwidth"
 278                     required:
 279                     -   type
 280                     -   bandwidth
 281         start_time:
 282             description: >-
 283                 UTC date and time in RFC 3339 format, specifying calendar time
 284                 just after test case started (at the start of test setup).
 285             $ref: "#/$defs/types/date_time"
 286         tags:
 287             description: >-
 288                 The list of strings comes directly
 289                 from Robot variable TEST_TAGS.
 290                 The content should include both static and dynamic tags
 291                 at the end of test case (teardown).
 292             type: array
 293             items:
 294                 type: string
 295         test_documentation:
 296             description: >-
 297                 Value taken directly from TEST_DOCUMENTATION Robot variable.
 298                 The content is what you see in suite file
 299                 at test case definition, which is usually empty
 300                 as CSIT uses data driven test cases.
 301             type: string
 302         test_id:
 303             description: >-
 304                 A derived quantity.
 305                 It is the most complete and unique identifier for a test case.
 306                 This property has a value, of the following form:
 307                 {suite_name}.{test_name}
 308                 Here, suite name comes from SUITE_NAME robot variable,
 309                 test name comes from TEST_NAME robot variable,
 310                 but both are converted to lower case,
 311                 and spaces are replaced by underscores.
 312             type: string
 313             minLength: 3
 314         test_name_long:
 315             description: >-
 316                 A derived quantity.
 317                 This property has a value, of the following form:
 318                 {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part}
 319                 Here, suite part is very similar to suite tag,
 320                 but additionally may contain a prefix describing NIC driver used
 321                 (if it is not the default one, drv_vfio_pci for VPP tests).
 322                 Any space is replaced by underscore and letters are lower case.
 323             type: string
 324             minLength: 3
 325         test_name_short:
 326             description: >-
 327                 A derived quantity.
 328                 This property has a value very similar to suite tag,
 329                 but additionally may contain a prefix describing NIC driver used
 330                 (if it is not the default one, drv_vfio_pci for VPP tests).
 331                 Any space is replaced by underscore and letters are lower case.
 332             type: string
 333             minLength: 3
 334         test_type:
 335             description: >-
 336                 A derived quantity.
 337                 Test type identifier, PAL uses it to group similar tests,
 338                 e.g. for comparison tables.
 339                 Ideally, this information should be parseable from test name,
 340                 but the current naming scheme is not simple/consistent enough.
 341                 The current implementation queries the robot test tags.
 342                 The resulting value is frequently identical to result type,
 343                 but this schema version does not require any relation there,
 344                 as PAL may want to group tests differently.
 345             type: string
 346             enum:
 347             -   device
 348             -   gso
 349             -   hoststack
 350             -   mrr
 351             -   ndrpdr
 352             -   reconf
 353             -   soak
 354         tg_type:
 355             description: >-
 356                 TG type used, e.g. TREX.
 357             type: string
 358             minLength: 1
 359         tg_version:
 360             description: >-
 361                 Version string appropriate to TG type used.
 362             type: string
 363             minLength: 1
 364         version:
 365             description: >-
 366                 CSIT model version (semver format)
 367                 the exporting code adhered to.
 368             type: string
 369             const: 1.5.1
 370     required:
 371     -   duration
 372     -   dut_type
 373     -   dut_version
 374     -   end_time
 375     -   hosts
 376     -   telemetry
 377     -   message
 378     -   passed
 379     -   result
 380     -   start_time
 381     -   tags
 382     -   test_documentation
 383     -   test_id
 384     -   test_name_long
 385     -   test_name_short
 386     -   test_type
 387     -   tg_type
 388     -   tg_version
 389     -   version
 390 -   description: >-
 391         Subschema validating relation between status and message.
 392     oneOf:
 393     -   description: >-
 394             Subschema for passing tests, message has to be empty.
 395         type: object
 396         properties:
 397             passed:
 398                 const: true
 399             message:
 400                 const: ""
 401     -   description: >-
 402             Subschema for failing tests, mesage cannot be empty.
 403         type: object
 404         properties:
 405             passed:
 406                 const: false
 407             message:
 408                 minLength: 1
 409
 410 $defs:
 411     types:
 412         nonnegative_number:
 413             type: number
 414             minimum: 0
 415         positive_number:
 416             type: number
 417             minimum: 1
 418         nonnegative_integer:
 419             type: integer
 420             minimum: 0
 421         positive_integer:
 422             type: integer
 423             minimum: 1
 424         date_time:
 425             type: string
 426             format: date-time
 427         empty_array:
 428             type: array
 429             maxItems: 0
 430         rate_unit:
 431             description: >-
 432                 Packets per second (pps),
 433                 connections per second (cps),
 434                 requests per second (rps),
 435                 bits per second (bps).
 436             type: string
 437             enum:
 438             -   pps
 439             -   cps
 440             -   rps
 441             -   bps
 442         bandwidth_unit:
 443             description: >-
 444                 Unit of measurement for bandwidth values.
 445                 Currently a constant, but later versions of model
 446                 may allow more units.
 447             enum:
 448             -   bps
 449         count_packets:
 450             description: >-
 451                 Type, for counting packets.
 452             allOf:
 453             -   $ref: "#/$defs/types/value_with_unit"
 454             -   properties:
 455                     value:
 456                         description: >-
 457                             A number of packets of interest.
 458                     unit:
 459                         description: >-
 460                             Unit suitable for displaying packet counts.
 461                         enum:
 462                         -   packets
 463         count_requests:
 464             description: >-
 465                 Type, for counting requests.
 466             allOf:
 467             -   $ref: "#/$defs/types/value_with_unit"
 468             -   properties:
 469                     value:
 470                         description: >-
 471                             A number of requests of interest.
 472                     unit:
 473                         description: >-
 474                             Unit suitable for displaying request counts.
 475                         enum:
 476                         -   requests
 477         time_quantity:
 478             description: >-
 479                 Reusable type, for various time quantites.
 480             allOf:
 481             -   $ref: "#/$defs/types/value_with_unit"
 482             -   properties:
 483                     value:
 484                         description: >-
 485                             Unless specified otherwise, this is a duration
 486                             between two events.
 487                     unit:
 488                         description: >-
 489                             Only seconds are the unit supported for time
 490                             quantities.
 491                         enum:
 492                         -   s
 493         value_with_unit:
 494             description: >-
 495                 Reusable composite type, value together with its
 496                 unit of measurement.
 497             type: object
 498             additionalProperties: false
 499             properties:
 500                 value:
 501                     description: >-
 502                         Numeric value, context specified elsewhere.
 503                         The only assumption is that value is not negative.
 504                     $ref: "#/$defs/types/nonnegative_number"
 505                 unit:
 506                     description: >-
 507                         Unit of measurement for the value.
 508                         Context and allowed values are specified elsewhere.
 509                     type: string
 510             required:
 511             -   value
 512             -   unit
 513         rate_without_bandwidth:
 514             description: >-
 515                 Reusable type, for various rate quantites.
 516             allOf:
 517             -   $ref: "#/$defs/types/value_with_unit"
 518             -   properties:
 519                     value:
 520                         description: >-
 521                             Unless specified otherwise,
 522                             this is the aggregated rate
 523                             (sum of both traffic directions).
 524                             Depending on the usage, the value can express
 525                             intended load, offered load, receive rate,
 526                             and various approximations
 527                             or estimated bounds thereof.
 528                     unit:
 529                         description: >-
 530                             A transaction rate unit the value is expressed in.
 531                         $ref: "#/$defs/types/rate_unit"
 532         bandwidth:
 533             description: >-
 534                 Reusable type, for various bandwidth quantites.
 535             allOf:
 536             -   $ref: "#/$defs/types/value_with_unit"
 537             -   properties:
 538                     value:
 539                         description: >-
 540                             Bandwidth value computed from the corresponding
 541                             rate.
 542                     unit:
 543                         $ref: "#/$defs/types/bandwidth_unit"
 544         rate_with_bandwidth:
 545             description: >-
 546                 Reusable composite type, joining primary rate
 547                 with optional derived bandwidth.
 548                 Not all test types currently compute bandwidth,
 549                 even if rate unit is pps.
 550             type: object
 551             additionalProperties: false
 552             properties:
 553                 rate:
 554                     $ref: "#/$defs/types/rate_without_bandwidth"
 555                 bandwidth:
 556                     $ref: "#/$defs/types/bandwidth"
 557             required:
 558             - rate
 559         packet_with_time:
 560             description: >-
 561                 Reusable composite type, joining packet count with the
 562                 time quantity.
 563             type: object
 564             additionalProperties: false
 565             properties:
 566                 packet:
 567                     $ref: "#/$defs/types/count_packets"
 568                 time:
 569                     $ref: "#/$defs/types/time_quantity"
 570             required:
 571             - packet
 572             - time
 573         value_list_with_unit_and_stats:
 574             description: >-
 575                 Reusable composite type, multiple values together with their
 576                 unit of measurement and derived statistics.
 577             type: object
 578             additionalProperties: false
 579             properties:
 580                 values:
 581                     description: >-
 582                         List of values of the same unit, useful for MRR.
 583                     type: array
 584                     minItmes: 1
 585                     items:
 586                         description: >-
 587                             Numeric value, context specified elsewhere. The only
 588                             assumption is that the value is nonnegative.
 589                         $ref: "#/$defs/types/nonnegative_number"
 590                 avg:
 591                     description: >-
 592                         A derived quantity. It is the arithmetic average of the
 593                         values list.
 594                     $ref: "#/$defs/types/nonnegative_number"
 595                 stdev:
 596                     description: >-
 597                         A derived quantity. It is the standard deviation for the
 598                         values list, as computed by jumpavg library.
 599                     $ref: "#/$defs/types/nonnegative_number"
 600                 unit:
 601                     description: >-
 602                         Unit of measurement for the values.
 603                         Context and allowed values are specified elsewhere.
 604                     type: string
 605             required:
 606             -   values
 607             -   avg
 608             -   stdev
 609             -   unit
 610         rate_list_without_bandwidth:
 611             description: >-
 612                 Reusable composite type, multiple rate values.
 613             allOf:
 614             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 615             -   properties:
 616                     values:
 617                         items:
 618                             description: >-
 619                                 Unless specified otherwise,
 620                                 this is the aggregated rate
 621                                 (sum of both traffic directions).
 622                                 Depending on the usage, the value can express
 623                                 intended load, offered load, receive rate,
 624                                 and various approximations or estimated bounds
 625                                 thereof.
 626                     unit:
 627                         $ref: "#/$defs/types/rate_unit"
 628         bandwidth_list:
 629             description: >-
 630                 Reusable composite type, multiple bandwidth values. This is a
 631                 derived quantity.
 632             allOf:
 633             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 634             -   properties:
 635                     values:
 636                         items:
 637                             description: >-
 638                                 Unless specified otherwise,
 639                                 this is the aggregated bandwidth
 640                                 (sum of both traffic directions).
 641                                 Depending on the usage, the value can express
 642                                 intended load, offered load, receive rate,
 643                                 and various approximations or estimated bounds
 644                                 thereof.
 645                     unit:
 646                         $ref: "#/$defs/types/bandwidth_unit"
 647         rate_list_with_bandwidth:
 648             description: >-
 649                 Reusable composite type, joining primary rates
 650                 with optional derived bandwidths (and stats).
 651                 No test types currently computes the bandwidth part.
 652             type: object
 653             additionalProperties: false
 654             properties:
 655                 rate:
 656                     $ref: "#/$defs/types/rate_list_without_bandwidth"
 657                 bandwidth:
 658                     $ref: "#/$defs/types/bandwidth_list"
 659             required:
 660             - rate
 661     macros:
 662         lower_and_maybe_upper_rate:
 663             type: object
 664             additionalProperties: false
 665             properties:
 666                 lower:
 667                     description: >-
 668                         The lower bound (or min_rate) for the estimate
 669                         of a particular searched value.
 670                     $ref: "#/$defs/types/rate_with_bandwidth"
 671                 upper:
 672                     description: >-
 673                         The upper bound (or max_rate) for the estimate
 674                         of a particular searched value.
 675                     $ref: "#/$defs/types/rate_with_bandwidth"
 676             required:
 677             -   lower
 678         latency_numbers:
 679             type: object
 680             additionalProperties: false
 681             properties:
 682                 min:
 683                     description: >-
 684                         Rounded minimal latency time measured in this trial.
 685                         See unit property for the unit of measurement.
 686                     $ref: "#/$defs/types/nonnegative_integer"
 687                 max:
 688                     description: >-
 689                         Rounded maximal latency time measured in this trial.
 690                         See unit property for the unit of measurement.
 691                         Zero value is not allowed, as that is one of symptoms
 692                         of Traffic Generator failing to get proper latency.
 693                     $ref: "#/$defs/types/positive_integer"
 694                 avg:
 695                     description: >-
 696                         Rounded average latency time measured in this trial.
 697                         See unit property for the unit of measurement.
 698                     $ref: "#/$defs/types/nonnegative_integer"
 699                 hdrh:
 700                     description: >-
 701                         Base64-encoded compressed representation of HDRHistogram
 702                         of all latency sample times encountered
 703                         in this latency trial.
 704                         See unit property for the unit of measurement.
 705                         Note that some bins can be several units wide.
 706                     type: string
 707                 unit:
 708                     description: >-
 709                         Unit of measurement for latency times.
 710                         Currently a constant, but later versions
 711                         of the model may allow more values.
 712                     type: string
 713                     enum:
 714                     -   us
 715             required:
 716             -   avg
 717             -   hdrh
 718             -   max
 719             -   min
 720             -   unit
 721         latency_for_loads:
 722             type: object
 723             additionalProperties: false
 724             properties:
 725                 pdr_0:
 726                     description: >-
 727                         Object related to latency measurement performed
 728                         at minimal rate (currently 9000 pps per direction).
 729                     $ref: "#/$defs/macros/latency_numbers"
 730                 pdr_10:
 731                     description: >-
 732                         Object related to latency measurement performed
 733                         at 10% of PDR lower bound, if needed rounded up
 734                         to minimal rate (currently 9000 pps per direction).
 735                     $ref: "#/$defs/macros/latency_numbers"
 736                 pdr_50:
 737                     description: >-
 738                         Object related to latency measurement performed
 739                         at 50% of PDR lower bound, if needed rounded up
 740                         to minimal rate (currently 9000 pps per direction).
 741                     $ref: "#/$defs/macros/latency_numbers"
 742                 pdr_90:
 743                     description: >-
 744                         Object related to latency measurement performed
 745                         at 90% of PDR lower bound, if needed rounded up
 746                         to minimal rate (currently 9000 pps per direction).
 747                     $ref: "#/$defs/macros/latency_numbers"
 748             required:
 749             -   pdr_0
 750             -   pdr_10
 751             -   pdr_50
 752             -   pdr_90