resources/model_schema/test_case.schema.yaml

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 ---
  15
  16 $id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.4.0
  17 $schema: https://json-schema.org/draft/2020-12/schema
  18 description: >-
  19     Schema for output of test case.
  20 allOf:
  21 -   description: >-
  22         The main structure, without conditional relations between fields yet.
  23     type: object
  24     additionalProperties: false
  25     properties:
  26         duration:
  27             description: >-
  28                 A derived quantity. Difference between start_time and end_time,
  29                 in seconds.
  30             $ref: "#/$defs/types/nonnegative_number"
  31         dut_type:
  32             description: >-
  33                 DUT type used, e.g. VPP or DPDK.
  34             type: string
  35             minLength: 1
  36         dut_version:
  37             description: >-
  38                 Version string appropriate to DUT type used.
  39             type: string
  40             minLength: 1
  41         end_time:
  42             description: >-
  43                 UTC date and time in RFC 3339 format, specifying calendar time
  44                 just before test case ended (at the end of test case teardown).
  45             $ref: "#/$defs/types/date_time"
  46         hosts:
  47             description: >-
  48                 Array of hosts this test interacted with.
  49                 This can be used for identifying testbed number.
  50                 Valid tests shoud interact with at least one DUT or TG.
  51                 The array is usually sorted, but that is not a requirement.
  52             type: array
  53             minItems: 1
  54             items:
  55                 description: >-
  56                     Host identifier, usually numeric IPv4 address.
  57                 type: string
  58         telemetry:
  59             description: >-
  60                 Array of telemetry entries. Each entry represent one captured
  61                 metric.
  62             type: array
  63             minItems: 0
  64             items:
  65                 description: >-
  66                     Telemetry entry.
  67                 type: string
  68         message:
  69             description: >-
  70                 If passed is true, this value is empty.
  71                 Otherwise, value taken directly from TEST_MESSAGE
  72                 Robot variable, read at the end of test case
  73                 (in test teardown, before export and validation).
  74                 It contains information from the exception
  75                 that caused the failure, probably with additional
  76                 exceptions from teardown keywords.
  77             type: string
  78         passed:
  79             description: >-
  80                 Value set accordingly to TEST_STATUS Robot variable,
  81                 true if and only if the status is "PASS".
  82                 The status is read at the end of test case
  83                 (in test teardown, before export and validation).
  84             type: boolean
  85         result:
  86             type: object
  87             allOf:
  88             -   description: >-
  89                     Sub-schema common for all cases,
  90                     only result type identifier defined here.
  91                 properties:
  92                     type:
  93                         description: >-
  94                             Identifier of which result type case is applied.
  95                         type: string
  96                 required:
  97                 -   type
  98             -   oneOf:
  99                 -   description: >-
 100                         Result type for unknown case.
 101                         This case represents a test with no specific result
 102                         (outside message), e.g. device test;
 103                         or a test with result not parsed into
 104                         this version of model yet, e.g. GSO test.
 105                     additionalProperties: false
 106                     properties:
 107                         type:
 108                             const: unknown
 109                 -   description: >-
 110                         Result type MRR case.
 111                     additionalProperties: false
 112                     properties:
 113                         type:
 114                             const: mrr
 115                         receive_rate:
 116                             description: >-
 117                                 The results refer to receive rates for multiple
 118                                 MRR trials. For PPS, these are aggregate
 119                                 (bidirectional) rates.
 120                                 Currently, the tests are exporting
 121                                 approximated receive rates.
 122                                 That means the actual trial duration
 123                                 is measured (as opposed to trusting traffic
 124                                 generator to honor its target duration),
 125                                 so the resulting values contain noise
 126                                 from time measurement, and can be lower
 127                                 than the real performance
 128                                 (due to various time overheads).
 129                                 Bandwidth values are supported, but currently
 130                                 Robot does not export them.
 131                             $ref: "#/$defs/types/rate_list_with_bandwidth"
 132                     required:
 133                     -   type
 134                     -   receive_rate
 135                 -   description: >-
 136                         Result type NDRPDR case.
 137                     additionalProperties: false
 138                     properties:
 139                         type:
 140                             const: ndrpdr
 141                         ndr:
 142                             description: >-
 143                                 The results refer to search for NDR
 144                                 (Non Drop Rate). For PPS, this is aggregate
 145                                 (bidirectional) rate.
 146                                 Each bound was used as the target load value
 147                                 in a full-duration trial measurement.
 148                                 The accepted loss ratio for NDR is exact zero.
 149                                 Note that packets the Traffic Generator
 150                                 did not send are also counted as lost packets.
 151                             $ref: "#/$defs/macros/lower_and_upper_rate"
 152                         pdr:
 153                             description: >-
 154                                 The results refer to search for PDR
 155                                 (Partial Drop Rate). For PPS, this is aggregate
 156                                 (bidirectional) rate.
 157                                 Each bound was used as the target load value
 158                                 in a full-duration trial measurement.
 159                                 The accepted loss ratio for PDR is 0.5%.
 160                                 Note that packets the Traffic Generator
 161                                 did not send are also counted as lost packets.
 162                             $ref: "#/$defs/macros/lower_and_upper_rate"
 163                         latency_forward:
 164                             description: >-
 165                                 Object with results related to latency part
 166                                 of NDRPDR test, for forward traffic direction.
 167                                 It is the direction used in unidirectional
 168                                 traffic profiles.
 169                                 ASTF profiles and IMIX STL profiles
 170                                 do not support latency information,
 171                                 so for those tests this object is missing.
 172                                 It is also missing if Traffic Generator
 173                                 fails to return valid latency results
 174                                 for any other reasons,
 175                                 e.g. latency rate is too high for CPU/NIC used.
 176                             $ref: "#/$defs/macros/latency_for_loads"
 177                         latency_reverse:
 178                             description: >-
 179                                 Object with results related to latency part
 180                                 of NDRPDR test, for reverse traffic diration.
 181                                 This object is not present
 182                                 when unidirectional traffic profiles are used.
 183                                 ASTF profiles and IMIX STL profiles
 184                                 do not support latency information,
 185                                 so for those tests this object is missing.
 186                                 It is also missing if Traffic Generator
 187                                 fails to return valid latency results
 188                                 for any other reasons,
 189                                 e.g. latency rate is too high for CPU/NIC used.
 190                             $ref: "#/$defs/macros/latency_for_loads"
 191                     required:
 192                     -   type
 193                     -   ndr
 194                     -   pdr
 195                 -   description: >-
 196                         Result type SOAK case.
 197                     additionalProperties: false
 198                     properties:
 199                         type:
 200                             const: soak
 201                         critical_rate:
 202                             description: >-
 203                                 The results refer to bayesian estimate
 204                                 of critical rate corresponding to
 205                                 average loss ratio of 10^-7.
 206                                 For PPS, this is aggregate (bidirectional) rate.
 207                                 The bounds are computed from
 208                                 trial measurement results,
 209                                 but are not equal to any target load used.
 210                                 Note that packets the Traffic Generator
 211                                 did not send are also counted as lost packets.
 212                             $ref: "#/$defs/macros/lower_and_upper_rate"
 213                     required:
 214                     -   type
 215                     -   critical_rate
 216                 -   description: >-
 217                         Result type RECONF case.
 218                     additionalProperties: false
 219                     properties:
 220                         type:
 221                             const: reconf
 222                         aggregate_rate:
 223                             description: >-
 224                                 Load used when reconfiguring, found as NDR lower
 225                                 bound. This is an aggregate (bidirectional)
 226                                 rate. Note that packets which the Traffic
 227                                 Generator did not send, are also counted as lost
 228                                 packets.
 229                             $ref: "#/$defs/types/rate_with_bandwidth"
 230                         loss:
 231                             description: >-
 232                                 Number of packets lost during reconfiguration,
 233                                 with the time that equals packet loss divided by
 234                                 packet rate.
 235                             $ref: "#/$defs/types/packet_with_time"
 236                     required:
 237                     -   type
 238                     -   loss
 239                     -   aggregate_rate
 240                 -   description: >-
 241                         Result type HOSTSTACK case.
 242                     additionalProperties: false
 243                     properties:
 244                         type:
 245                             const: hoststack
 246                         bandwidth:
 247                             description: >-
 248                                 Goodput measured in bits per second.
 249                             $ref: "#/$defs/types/bandwidth"
 250                         completed_requests:
 251                             description: >-
 252                                 Number of completed requests.
 253                             $ref: "#/$defs/types/count_requests"
 254                         failed_requests:
 255                             description: >-
 256                                 Number of failed requests.
 257                             $ref: "#/$defs/types/count_requests"
 258                         retransmits:
 259                             description: >-
 260                                 Number of retransmits.
 261                             $ref: "#/$defs/types/count_packets"
 262                         latency:
 263                             description: >-
 264                                 Value and unit of latency.
 265                             $ref: "#/$defs/types/value_with_unit"
 266                         duration:
 267                             description: >-
 268                                 The relative time difference (in seconds)
 269                                 between program start and end.
 270                             $ref: "#/$defs/types/time_quantity"
 271                         rate:
 272                             description: >-
 273                                 RPS or CPS rate, with corresponding unit, as
 274                                 reported by TG.
 275                             $ref: "#/$defs/types/rate_without_bandwidth"
 276                     required:
 277                     -   type
 278                     -   bandwidth
 279         start_time:
 280             description: >-
 281                 UTC date and time in RFC 3339 format, specifying calendar time
 282                 just after test case started (at the start of test setup).
 283             $ref: "#/$defs/types/date_time"
 284         tags:
 285             description: >-
 286                 The list of strings comes directly
 287                 from Robot variable TEST_TAGS.
 288                 The content should include both static and dynamic tags
 289                 at the end of test case (teardown).
 290             type: array
 291             items:
 292                 type: string
 293         test_documentation:
 294             description: >-
 295                 Value taken directly from TEST_DOCUMENTATION Robot variable.
 296                 The content is what you see in suite file
 297                 at test case definition, which is usually empty
 298                 as CSIT uses data driven test cases.
 299             type: string
 300         test_id:
 301             description: >-
 302                 A derived quantity.
 303                 It is the most complete and unique identifier for a test case.
 304                 This property has a value, of the following form:
 305                 {suite_name}.{test_name}
 306                 Here, suite name comes from SUITE_NAME robot variable,
 307                 test name comes from TEST_NAME robot variable,
 308                 but both are converted to lower case,
 309                 and spaces are replaced by underscores.
 310             type: string
 311             minLength: 3
 312         test_name_long:
 313             description: >-
 314                 A derived quantity.
 315                 This property has a value, of the following form:
 316                 {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part}
 317                 Here, suite part is very similar to suite tag,
 318                 but additionally may contain a prefix describing NIC driver used
 319                 (if it is not the default one, drv_vfio_pci for VPP tests).
 320                 Any space is replaced by underscore and letters are lower case.
 321             type: string
 322             minLength: 3
 323         test_name_short:
 324             description: >-
 325                 A derived quantity.
 326                 This property has a value very similar to suite tag,
 327                 but additionally may contain a prefix describing NIC driver used
 328                 (if it is not the default one, drv_vfio_pci for VPP tests).
 329                 Any space is replaced by underscore and letters are lower case.
 330             type: string
 331             minLength: 3
 332         test_type:
 333             description: >-
 334                 A derived quantity.
 335                 Test type identifier, PAL uses it to group similar tests,
 336                 e.g. for comparison tables.
 337                 Ideally, this information should be parseable from test name,
 338                 but the current naming scheme is not simple/consistent enough.
 339                 The current implementation queries the robot test tags.
 340                 The resulting value is frequently identical to result type,
 341                 but this schema version does not require any relation there,
 342                 as PAL may want to group tests differently.
 343             type: string
 344             enum:
 345             -   device
 346             -   gso
 347             -   hoststack
 348             -   mrr
 349             -   ndrpdr
 350             -   reconf
 351             -   soak
 352         tg_type:
 353             description: >-
 354                 TG type used, e.g. TREX.
 355             type: string
 356             minLength: 1
 357         tg_version:
 358             description: >-
 359                 Version string appropriate to TG type used.
 360             type: string
 361             minLength: 1
 362         version:
 363             description: >-
 364                 CSIT model version (semver format)
 365                 the exporting code adhered to.
 366             type: string
 367             const: 1.4.0
 368     required:
 369     -   duration
 370     -   dut_type
 371     -   dut_version
 372     -   end_time
 373     -   hosts
 374     -   telemetry
 375     -   message
 376     -   passed
 377     -   result
 378     -   start_time
 379     -   tags
 380     -   test_documentation
 381     -   test_id
 382     -   test_name_long
 383     -   test_name_short
 384     -   test_type
 385     -   tg_type
 386     -   tg_version
 387     -   version
 388 -   description: >-
 389         Subschema validating relation between status and message.
 390     oneOf:
 391     -   description: >-
 392             Subschema for passing tests, message has to be empty.
 393         type: object
 394         properties:
 395             passed:
 396                 const: true
 397             message:
 398                 const: ""
 399     -   description: >-
 400             Subschema for failing tests, mesage cannot be empty.
 401         type: object
 402         properties:
 403             passed:
 404                 const: false
 405             message:
 406                 minLength: 1
 407
 408 $defs:
 409     types:
 410         nonnegative_number:
 411             type: number
 412             minimum: 0
 413         positive_number:
 414             type: number
 415             minimum: 1
 416         nonnegative_integer:
 417             type: integer
 418             minimum: 0
 419         positive_integer:
 420             type: integer
 421             minimum: 1
 422         date_time:
 423             type: string
 424             format: date-time
 425         empty_array:
 426             type: array
 427             maxItems: 0
 428         rate_unit:
 429             description: >-
 430                 Packets per second (pps),
 431                 connections per second (cps),
 432                 requests per second (rps),
 433                 bits per second (bps).
 434             type: string
 435             enum:
 436             -   pps
 437             -   cps
 438             -   rps
 439             -   bps
 440         bandwidth_unit:
 441             description: >-
 442                 Unit of measurement for bandwidth values.
 443                 Currently a constant, but later versions of model
 444                 may allow more units.
 445             enum:
 446             -   bps
 447         count_packets:
 448             description: >-
 449                 Type, for counting packets.
 450             allOf:
 451             -   $ref: "#/$defs/types/value_with_unit"
 452             -   properties:
 453                     value:
 454                         description: >-
 455                             A number of packets of interest.
 456                     unit:
 457                         description: >-
 458                             Unit suitable for displaying packet counts.
 459                         enum:
 460                         -   packets
 461         count_requests:
 462             description: >-
 463                 Type, for counting requests.
 464             allOf:
 465             -   $ref: "#/$defs/types/value_with_unit"
 466             -   properties:
 467                     value:
 468                         description: >-
 469                             A number of requests of interest.
 470                     unit:
 471                         description: >-
 472                             Unit suitable for displaying request counts.
 473                         enum:
 474                         -   requests
 475         time_quantity:
 476             description: >-
 477                 Reusable type, for various time quantites.
 478             allOf:
 479             -   $ref: "#/$defs/types/value_with_unit"
 480             -   properties:
 481                     value:
 482                         description: >-
 483                             Unless specified otherwise, this is a duration
 484                             between two events.
 485                     unit:
 486                         description: >-
 487                             Only seconds are the unit supported for time
 488                             quantities.
 489                         enum:
 490                         -   s
 491         value_with_unit:
 492             description: >-
 493                 Reusable composite type, value together with its
 494                 unit of measurement.
 495             type: object
 496             additionalProperties: false
 497             properties:
 498                 value:
 499                     description: >-
 500                         Numeric value, context specified elsewhere.
 501                         The only assumption is that value is not negative.
 502                     $ref: "#/$defs/types/nonnegative_number"
 503                 unit:
 504                     description: >-
 505                         Unit of measurement for the value.
 506                         Context and allowed values are specified elsewhere.
 507                     type: string
 508             required:
 509             -   value
 510             -   unit
 511         rate_without_bandwidth:
 512             description: >-
 513                 Reusable type, for various rate quantites.
 514             allOf:
 515             -   $ref: "#/$defs/types/value_with_unit"
 516             -   properties:
 517                     value:
 518                         description: >-
 519                             Unless specified otherwise,
 520                             this is the aggregated rate
 521                             (sum of both traffic directions).
 522                             Depending on the usage, the value can express
 523                             intended load, offered load, receive rate,
 524                             and various approximations
 525                             or estimated bounds thereof.
 526                     unit:
 527                         description: >-
 528                             A transaction rate unit the value is expressed in.
 529                         $ref: "#/$defs/types/rate_unit"
 530         bandwidth:
 531             description: >-
 532                 Reusable type, for various bandwidth quantites.
 533             allOf:
 534             -   $ref: "#/$defs/types/value_with_unit"
 535             -   properties:
 536                     value:
 537                         description: >-
 538                             Bandwidth value computed from the corresponding
 539                             rate.
 540                     unit:
 541                         $ref: "#/$defs/types/bandwidth_unit"
 542         rate_with_bandwidth:
 543             description: >-
 544                 Reusable composite type, joining primary rate
 545                 with optional derived bandwidth.
 546                 Not all test types currently compute bandwidth,
 547                 even if rate unit is pps.
 548             type: object
 549             additionalProperties: false
 550             properties:
 551                 rate:
 552                     $ref: "#/$defs/types/rate_without_bandwidth"
 553                 bandwidth:
 554                     $ref: "#/$defs/types/bandwidth"
 555             required:
 556             - rate
 557         packet_with_time:
 558             description: >-
 559                 Reusable composite type, joining packet count with the
 560                 time quantity.
 561             type: object
 562             additionalProperties: false
 563             properties:
 564                 packet:
 565                     $ref: "#/$defs/types/count_packets"
 566                 time:
 567                     $ref: "#/$defs/types/time_quantity"
 568             required:
 569             - packet
 570             - time
 571         value_list_with_unit_and_stats:
 572             description: >-
 573                 Reusable composite type, multiple values together with their
 574                 unit of measurement and derived statistics.
 575             type: object
 576             additionalProperties: false
 577             properties:
 578                 values:
 579                     description: >-
 580                         List of values of the same unit, useful for MRR.
 581                     type: array
 582                     minItmes: 1
 583                     items:
 584                         description: >-
 585                             Numeric value, context specified elsewhere. The only
 586                             assumption is that the value is nonnegative.
 587                         $ref: "#/$defs/types/nonnegative_number"
 588                 avg:
 589                     description: >-
 590                         A derived quantity. It is the arithmetic average of the
 591                         values list.
 592                     $ref: "#/$defs/types/nonnegative_number"
 593                 stdev:
 594                     description: >-
 595                         A derived quantity. It is the standard deviation for the
 596                         values list, as computed by jumpavg library.
 597                     $ref: "#/$defs/types/nonnegative_number"
 598                 unit:
 599                     description: >-
 600                         Unit of measurement for the values.
 601                         Context and allowed values are specified elsewhere.
 602                     type: string
 603             required:
 604             -   values
 605             -   avg
 606             -   stdev
 607             -   unit
 608         rate_list_without_bandwidth:
 609             description: >-
 610                 Reusable composite type, multiple rate values.
 611             allOf:
 612             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 613             -   properties:
 614                     values:
 615                         items:
 616                             description: >-
 617                                 Unless specified otherwise,
 618                                 this is the aggregated rate
 619                                 (sum of both traffic directions).
 620                                 Depending on the usage, the value can express
 621                                 intended load, offered load, receive rate,
 622                                 and various approximations or estimated bounds
 623                                 thereof.
 624                     unit:
 625                         $ref: "#/$defs/types/rate_unit"
 626         bandwidth_list:
 627             description: >-
 628                 Reusable composite type, multiple bandwidth values. This is a
 629                 derived quantity.
 630             allOf:
 631             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 632             -   properties:
 633                     values:
 634                         items:
 635                             description: >-
 636                                 Unless specified otherwise,
 637                                 this is the aggregated bandwidth
 638                                 (sum of both traffic directions).
 639                                 Depending on the usage, the value can express
 640                                 intended load, offered load, receive rate,
 641                                 and various approximations or estimated bounds
 642                                 thereof.
 643                     unit:
 644                         $ref: "#/$defs/types/bandwidth_unit"
 645         rate_list_with_bandwidth:
 646             description: >-
 647                 Reusable composite type, joining primary rates
 648                 with optional derived bandwidths (and stats).
 649                 No test types currently computes the bandwidth part.
 650             type: object
 651             additionalProperties: false
 652             properties:
 653                 rate:
 654                     $ref: "#/$defs/types/rate_list_without_bandwidth"
 655                 bandwidth:
 656                     $ref: "#/$defs/types/bandwidth_list"
 657             required:
 658             - rate
 659     macros:
 660         lower_and_upper_rate:
 661             type: object
 662             additionalProperties: false
 663             properties:
 664                 lower:
 665                     description: >-
 666                         The lower bound (or min_rate) for the estimate
 667                         of a particular searched value.
 668                     $ref: "#/$defs/types/rate_with_bandwidth"
 669                 upper:
 670                     description: >-
 671                         The upper bound (or max_rate) for the estimate
 672                         of a particular searched value.
 673                     $ref: "#/$defs/types/rate_with_bandwidth"
 674             required:
 675             -   lower
 676             -   upper
 677         latency_numbers:
 678             type: object
 679             additionalProperties: false
 680             properties:
 681                 min:
 682                     description: >-
 683                         Rounded minimal latency time measured in this trial.
 684                         See unit property for the unit of measurement.
 685                     $ref: "#/$defs/types/nonnegative_integer"
 686                 max:
 687                     description: >-
 688                         Rounded maximal latency time measured in this trial.
 689                         See unit property for the unit of measurement.
 690                         Zero value is not allowed, as that is one of symptoms
 691                         of Traffic Generator failing to get proper latency.
 692                     $ref: "#/$defs/types/positive_integer"
 693                 avg:
 694                     description: >-
 695                         Rounded average latency time measured in this trial.
 696                         See unit property for the unit of measurement.
 697                     $ref: "#/$defs/types/nonnegative_integer"
 698                 hdrh:
 699                     description: >-
 700                         Base64-encoded compressed representation of HDRHistogram
 701                         of all latency sample times encountered
 702                         in this latency trial.
 703                         See unit property for the unit of measurement.
 704                         Note that some bins can be several units wide.
 705                     type: string
 706                 unit:
 707                     description: >-
 708                         Unit of measurement for latency times.
 709                         Currently a constant, but later versions
 710                         of the model may allow more values.
 711                     type: string
 712                     enum:
 713                     -   us
 714             required:
 715             -   avg
 716             -   hdrh
 717             -   max
 718             -   min
 719             -   unit
 720         latency_for_loads:
 721             type: object
 722             additionalProperties: false
 723             properties:
 724                 pdr_0:
 725                     description: >-
 726                         Object related to latency measurement performed
 727                         at minimal rate (currently 9000 pps per direction).
 728                     $ref: "#/$defs/macros/latency_numbers"
 729                 pdr_10:
 730                     description: >-
 731                         Object related to latency measurement performed
 732                         at 10% of PDR lower bound, if needed rounded up
 733                         to minimal rate (currently 9000 pps per direction).
 734                     $ref: "#/$defs/macros/latency_numbers"
 735                 pdr_50:
 736                     description: >-
 737                         Object related to latency measurement performed
 738                         at 50% of PDR lower bound, if needed rounded up
 739                         to minimal rate (currently 9000 pps per direction).
 740                     $ref: "#/$defs/macros/latency_numbers"
 741                 pdr_90:
 742                     description: >-
 743                         Object related to latency measurement performed
 744                         at 90% of PDR lower bound, if needed rounded up
 745                         to minimal rate (currently 9000 pps per direction).
 746                     $ref: "#/$defs/macros/latency_numbers"
 747             required:
 748             -   pdr_0
 749             -   pdr_10
 750             -   pdr_50
 751             -   pdr_90