docs/model/current/schema/test_case.info.schema.yaml

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 ---
  15
  16 $id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.3.0
  17 $schema: https://json-schema.org/draft/2020-12/schema
  18 description: >-
  19     Schema for output of test case.
  20 allOf:
  21 -   description: >-
  22         The main structure, without conditional relations between fields yet.
  23     type: object
  24     additionalProperties: false
  25     properties:
  26         duration:
  27             description: >-
  28                 A derived quantity. Difference between start_time and end_time,
  29                 in seconds.
  30             $ref: "#/$defs/types/nonnegative_number"
  31         dut_type:
  32             description: >-
  33                 DUT type used, e.g. VPP or DPDK.
  34             type: string
  35             minLength: 1
  36         dut_version:
  37             description: >-
  38                 Version string appropriate to DUT type used.
  39             type: string
  40             minLength: 1
  41         end_time:
  42             description: >-
  43                 UTC date and time in RFC 3339 format, specifying calendar time
  44                 just before test case ended (at the end of test case teardown).
  45             $ref: "#/$defs/types/date_time"
  46         hosts:
  47             description: >-
  48                 Array of hosts this test interacted with.
  49                 This can be used for identifying testbed number.
  50                 Valid tests shoud interact with at least one DUT or TG.
  51                 The array is usually sorted, but that is not a requirement.
  52             type: array
  53             minItems: 1
  54             items:
  55                 description: >-
  56                     Host identifier, usually numeric IPv4 address.
  57                 type: string
  58         telemetry:
  59             description: >-
  60                 Array of telemetry entries. Each entry represent one captured
  61                 metric.
  62             type: array
  63             minItems: 0
  64             items:
  65                 description: >-
  66                     Telemetry entry.
  67                 type: string
  68         message:
  69             description: >-
  70                 If passed is true, this value is empty.
  71                 Otherwise, value taken directly from TEST_MESSAGE
  72                 Robot variable, read at the end of test case
  73                 (in test teardown, before export and validation).
  74                 It contains information from the exception
  75                 that caused the failure, probably with additional
  76                 exceptions from teardown keywords.
  77             type: string
  78         passed:
  79             description: >-
  80                 Value set accordingly to TEST_STATUS Robot variable,
  81                 true if and only if the status is "PASS".
  82                 The status is read at the end of test case
  83                 (in test teardown, before export and validation).
  84             type: boolean
  85         result:
  86             type: object
  87             allOf:
  88             -   description: >-
  89                     Sub-schema common for all cases,
  90                     only result type identifier defined here.
  91                 properties:
  92                     type:
  93                         description: >-
  94                             Identifier of which result type case is applied.
  95                         type: string
  96                 required:
  97                 -   type
  98             -   oneOf:
  99                 -   description: >-
 100                         Result type for unknown case.
 101                         This case represents a test with no specific result
 102                         (outside message), e.g. device test;
 103                         or a test with result not parsed into
 104                         this version of model yet, e.g. GSO test.
 105                     additionalProperties: false
 106                     properties:
 107                         type:
 108                             const: unknown
 109                 -   description: >-
 110                         Result type MRR case.
 111                     additionalProperties: false
 112                     properties:
 113                         type:
 114                             const: mrr
 115                         receive_rate:
 116                             description: >-
 117                                 The results refer to receive rates for multiple
 118                                 MRR trials. For PPS, these are aggregate
 119                                 (bidirectional) rates.
 120                                 Currently, the tests are exporting
 121                                 approximated receive rates.
 122                                 That means the actual trial duration
 123                                 is measured (as opposed to trusting traffic
 124                                 generator to honor its target duration),
 125                                 so the resulting values contain noise
 126                                 from time measurement, and can be lower
 127                                 than the real performance
 128                                 (due to various time overheads).
 129                                 Bandwidth values are supported, but currently
 130                                 Robot does not export them.
 131                             $ref: "#/$defs/types/rate_list_with_bandwidth"
 132                     required:
 133                     -   type
 134                     -   receive_rate
 135                 -   description: >-
 136                         Result type NDRPDR case.
 137                     additionalProperties: false
 138                     properties:
 139                         type:
 140                             const: ndrpdr
 141                         ndr:
 142                             description: >-
 143                                 The results refer to search for NDR
 144                                 (Non Drop Rate). For PPS, this is aggregate
 145                                 (bidirectional) rate.
 146                                 Each bound was used as the target load value
 147                                 in a full-duration trial measurement.
 148                                 The accepted loss ratio for NDR is exact zero.
 149                                 Note that packets the Traffic Generator
 150                                 did not send are also counted as lost packets.
 151                             $ref: "#/$defs/macros/lower_and_upper_rate"
 152                         pdr:
 153                             description: >-
 154                                 The results refer to search for PDR
 155                                 (Partial Drop Rate). For PPS, this is aggregate
 156                                 (bidirectional) rate.
 157                                 Each bound was used as the target load value
 158                                 in a full-duration trial measurement.
 159                                 The accepted loss ratio for PDR is 0.5%.
 160                                 Note that packets the Traffic Generator
 161                                 did not send are also counted as lost packets.
 162                             $ref: "#/$defs/macros/lower_and_upper_rate"
 163                         latency_forward:
 164                             description: >-
 165                                 Object with results related to latency part
 166                                 of NDRPDR test, for forward traffic direction.
 167                                 It is the direction used in unidirectional
 168                                 traffic profiles.
 169                                 ASTF profiles and IMIX STL profiles
 170                                 do not support latency information,
 171                                 so for those tests this object is missing.
 172                                 It is also missing if Traffic Generator
 173                                 fails to return valid latency results
 174                                 for any other reasons,
 175                                 e.g. latency rate is too high for CPU/NIC used.
 176                             $ref: "#/$defs/macros/latency_for_loads"
 177                         latency_reverse:
 178                             description: >-
 179                                 Object with results related to latency part
 180                                 of NDRPDR test, for reverse traffic diration.
 181                                 This object is not present
 182                                 when unidirectional traffic profiles are used.
 183                                 ASTF profiles and IMIX STL profiles
 184                                 do not support latency information,
 185                                 so for those tests this object is missing.
 186                                 It is also missing if Traffic Generator
 187                                 fails to return valid latency results
 188                                 for any other reasons,
 189                                 e.g. latency rate is too high for CPU/NIC used.
 190                             $ref: "#/$defs/macros/latency_for_loads"
 191                     required:
 192                     -   type
 193                     -   ndr
 194                     -   pdr
 195                 -   description: >-
 196                         Result type SOAK case.
 197                     additionalProperties: false
 198                     properties:
 199                         type:
 200                             const: soak
 201                         critical_rate:
 202                             description: >-
 203                                 The results refer to bayesian estimate
 204                                 of critical rate corresponding to
 205                                 average loss ratio of 10^-7.
 206                                 For PPS, this is aggregate (bidirectional) rate.
 207                                 The bounds are computed from
 208                                 trial measurement results,
 209                                 but are not equal to any target load used.
 210                                 Note that packets the Traffic Generator
 211                                 did not send are also counted as lost packets.
 212                             $ref: "#/$defs/macros/lower_and_upper_rate"
 213                     required:
 214                     -   type
 215                     -   critical_rate
 216                 -   description: >-
 217                         Result type RECONF case.
 218                     additionalProperties: false
 219                     properties:
 220                         type:
 221                             const: reconf
 222                         aggregate_rate:
 223                             description: >-
 224                                 Load used when reconfiguring, found as NDR lower
 225                                 bound. This is an aggregate (bidirectional)
 226                                 rate. Note that packets which the Traffic
 227                                 Generator did not send, are also counted as lost
 228                                 packets.
 229                             $ref: "#/$defs/types/rate_with_bandwidth"
 230                         loss:
 231                             description: >-
 232                                 Number of packets lost during reconfiguration,
 233                                 with the time that equals packet loss divided by
 234                                 packet rate.
 235                             $ref: "#/$defs/types/packet_with_time"
 236                     required:
 237                     -   type
 238                     -   loss
 239                     -   aggregate_rate
 240                 -   description: >-
 241                         Result type HOSTSTACK case.
 242                     additionalProperties: false
 243                     properties:
 244                         type:
 245                             const: hoststack
 246                         bandwidth:
 247                             description: >-
 248                                 Goodput measured in bits per second.
 249                             $ref: "#/$defs/types/bandwidth"
 250                         completed_requests:
 251                             description: >-
 252                                 Number of completed requests.
 253                             $ref: "#/$defs/types/count_requests"
 254                         failed_requests:
 255                             description: >-
 256                                 Number of failed requests.
 257                             $ref: "#/$defs/types/count_requests"
 258                         retransmits:
 259                             description: >-
 260                                 Number of retransmits.
 261                             $ref: "#/$defs/types/count_packets"
 262                         latency:
 263                             description: >-
 264                                 Value and unit of latency.
 265                             $ref: "#/$defs/types/value_with_unit"
 266                         duration:
 267                             description: >-
 268                                 The relative time difference (in seconds)
 269                                 between program start and end.
 270                             $ref: "#/$defs/types/time_quantity"
 271                         rate:
 272                             description: >-
 273                                 RPS or CPS rate, with corresponding unit, as
 274                                 reported by TG.
 275                             $ref: "#/$defs/types/rate_without_bandwidth"
 276                     required:
 277                     -   type
 278                     -   bandwidth
 279         start_time:
 280             description: >-
 281                 UTC date and time in RFC 3339 format, specifying calendar time
 282                 just after test case started (at the start of test setup).
 283             $ref: "#/$defs/types/date_time"
 284         tags:
 285             description: >-
 286                 The list of strings comes directly
 287                 from Robot variable TEST_TAGS.
 288                 The content should include both static and dynamic tags
 289                 at the end of test case (teardown).
 290             type: array
 291             items:
 292                 type: string
 293         test_documentation:
 294             description: >-
 295                 Value taken directly from TEST_DOCUMENTATION Robot variable.
 296                 The content is what you see in suite file
 297                 at test case definition, which is usually empty
 298                 as CSIT uses data driven test cases.
 299             type: string
 300         test_id:
 301             description: >-
 302                 A derived quantity.
 303                 It is the most complete and unique identifier for a test case.
 304                 This property has a value, of the following form:
 305                 {suite_name}.{test_name}
 306                 Here, suite name comes from SUITE_NAME robot variable,
 307                 test name comes from TEST_NAME robot variable,
 308                 but both are converted to lower case,
 309                 and spaces are replaced by underscores.
 310             type: string
 311             minLength: 3
 312         test_name_long:
 313             description: >-
 314                 A derived quantity.
 315                 This property has a value, of the following form:
 316                 {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part}
 317                 Here, suite part is very similar to suite tag,
 318                 but additionally may contain a prefix describing NIC driver used
 319                 (if it is not the default one, drv_vfio_pci for VPP tests).
 320                 Any space is replaced by underscore and letters are lower case.
 321             type: string
 322             minLength: 3
 323         test_name_short:
 324             description: >-
 325                 A derived quantity.
 326                 This property has a value very similar to suite tag,
 327                 but additionally may contain a prefix describing NIC driver used
 328                 (if it is not the default one, drv_vfio_pci for VPP tests).
 329                 Any space is replaced by underscore and letters are lower case.
 330             type: string
 331             minLength: 3
 332         test_type:
 333             description: >-
 334                 A derived quantity.
 335                 Test type identifier, PAL uses it to group similar tests,
 336                 e.g. for comparison tables.
 337                 Ideally, this information should be parseable from test name,
 338                 but the current naming scheme is not simple/consistent enough.
 339                 The current implementation queries the robot test tags.
 340                 The resulting value is frequently identical to result type,
 341                 but this schema version does not require any relation there,
 342                 as PAL may want to group tests differently.
 343             type: string
 344             enum:
 345             -   device
 346             -   gso
 347             -   hoststack
 348             -   mrr
 349             -   ndrpdr
 350             -   reconf
 351             -   soak
 352             -   vsap
 353         tg_type:
 354             description: >-
 355                 TG type used, e.g. TREX.
 356             type: string
 357             minLength: 1
 358         tg_version:
 359             description: >-
 360                 Version string appropriate to TG type used.
 361             type: string
 362             minLength: 1
 363         version:
 364             description: >-
 365                 CSIT model version (semver format)
 366                 the exporting code adhered to.
 367             type: string
 368             const: 1.3.0
 369     required:
 370     -   duration
 371     -   dut_type
 372     -   dut_version
 373     -   end_time
 374     -   hosts
 375     -   telemetry
 376     -   message
 377     -   passed
 378     -   result
 379     -   start_time
 380     -   tags
 381     -   test_documentation
 382     -   test_id
 383     -   test_name_long
 384     -   test_name_short
 385     -   test_type
 386     -   tg_type
 387     -   tg_version
 388     -   version
 389 -   description: >-
 390         Subschema validating relation between status and message.
 391     oneOf:
 392     -   description: >-
 393             Subschema for passing tests, message has to be empty.
 394         type: object
 395         properties:
 396             passed:
 397                 const: true
 398             message:
 399                 const: ""
 400     -   description: >-
 401             Subschema for failing tests, mesage cannot be empty.
 402         type: object
 403         properties:
 404             passed:
 405                 const: false
 406             message:
 407                 minLength: 1
 408
 409 $defs:
 410     types:
 411         nonnegative_number:
 412             type: number
 413             minimum: 0
 414         positive_number:
 415             type: number
 416             minimum: 1
 417         nonnegative_integer:
 418             type: integer
 419             minimum: 0
 420         positive_integer:
 421             type: integer
 422             minimum: 1
 423         date_time:
 424             type: string
 425             format: date-time
 426         empty_array:
 427             type: array
 428             maxItems: 0
 429         rate_unit:
 430             description: >-
 431                 Packets per second (pps),
 432                 connections per second (cps),
 433                 requests per second (rps).
 434             type: string
 435             enum:
 436             -   pps
 437             -   cps
 438             -   rps
 439         bandwidth_unit:
 440             description: >-
 441                 Unit of measurement for bandwidth values.
 442                 Currently a constant, but later versions of model
 443                 may allow more units.
 444             enum:
 445             -   bps
 446         count_packets:
 447             description: >-
 448                 Type, for counting packets.
 449             allOf:
 450             -   $ref: "#/$defs/types/value_with_unit"
 451             -   properties:
 452                     value:
 453                         description: >-
 454                             A number of packets of interest.
 455                     unit:
 456                         description: >-
 457                             Unit suitable for displaying packet counts.
 458                         enum:
 459                         -   packets
 460         count_requests:
 461             description: >-
 462                 Type, for counting requests.
 463             allOf:
 464             -   $ref: "#/$defs/types/value_with_unit"
 465             -   properties:
 466                     value:
 467                         description: >-
 468                             A number of requests of interest.
 469                     unit:
 470                         description: >-
 471                             Unit suitable for displaying request counts.
 472                         enum:
 473                         -   requests
 474         time_quantity:
 475             description: >-
 476                 Reusable type, for various time quantites.
 477             allOf:
 478             -   $ref: "#/$defs/types/value_with_unit"
 479             -   properties:
 480                     value:
 481                         description: >-
 482                             Unless specified otherwise, this is a duration
 483                             between two events.
 484                     unit:
 485                         description: >-
 486                             Only seconds are the unit supported for time
 487                             quantities.
 488                         enum:
 489                         -   s
 490         value_with_unit:
 491             description: >-
 492                 Reusable composite type, value together with its
 493                 unit of measurement.
 494             type: object
 495             additionalProperties: false
 496             properties:
 497                 value:
 498                     description: >-
 499                         Numeric value, context specified elsewhere.
 500                         The only assumption is that value is not negative.
 501                     $ref: "#/$defs/types/nonnegative_number"
 502                 unit:
 503                     description: >-
 504                         Unit of measurement for the value.
 505                         Context and allowed values are specified elsewhere.
 506                     type: string
 507             required:
 508             -   value
 509             -   unit
 510         rate_without_bandwidth:
 511             description: >-
 512                 Reusable type, for various rate quantites.
 513             allOf:
 514             -   $ref: "#/$defs/types/value_with_unit"
 515             -   properties:
 516                     value:
 517                         description: >-
 518                             Unless specified otherwise,
 519                             this is the aggregated rate
 520                             (sum of both traffic directions).
 521                             Depending on the usage, the value can express
 522                             intended load, offered load, receive rate,
 523                             and various approximations
 524                             or estimated bounds thereof.
 525                     unit:
 526                         description: >-
 527                             A transaction rate unit the value is expressed in.
 528                         $ref: "#/$defs/types/rate_unit"
 529         bandwidth:
 530             description: >-
 531                 Reusable type, for various bandwidth quantites.
 532             allOf:
 533             -   $ref: "#/$defs/types/value_with_unit"
 534             -   properties:
 535                     value:
 536                         description: >-
 537                             Bandwidth value computed from the corresponding
 538                             rate.
 539                     unit:
 540                         $ref: "#/$defs/types/bandwidth_unit"
 541         rate_with_bandwidth:
 542             description: >-
 543                 Reusable composite type, joining primary rate
 544                 with optional derived bandwidth.
 545                 Not all test types currently compute bandwidth,
 546                 even if rate unit is pps.
 547             type: object
 548             additionalProperties: false
 549             properties:
 550                 rate:
 551                     $ref: "#/$defs/types/rate_without_bandwidth"
 552                 bandwidth:
 553                     $ref: "#/$defs/types/bandwidth"
 554             required:
 555             - rate
 556         packet_with_time:
 557             description: >-
 558                 Reusable composite type, joining packet count with the
 559                 time quantity.
 560             type: object
 561             additionalProperties: false
 562             properties:
 563                 packet:
 564                     $ref: "#/$defs/types/count_packets"
 565                 time:
 566                     $ref: "#/$defs/types/time_quantity"
 567             required:
 568             - packet
 569             - time
 570         value_list_with_unit_and_stats:
 571             description: >-
 572                 Reusable composite type, multiple values together with their
 573                 unit of measurement and derived statistics.
 574             type: object
 575             additionalProperties: false
 576             properties:
 577                 values:
 578                     description: >-
 579                         List of values of the same unit, useful for MRR.
 580                     type: array
 581                     minItmes: 1
 582                     items:
 583                         description: >-
 584                             Numeric value, context specified elsewhere. The only
 585                             assumption is that the value is nonnegative.
 586                         $ref: "#/$defs/types/nonnegative_number"
 587                 avg:
 588                     description: >-
 589                         A derived quantity. It is the arithmetic average of the
 590                         values list.
 591                     $ref: "#/$defs/types/nonnegative_number"
 592                 stdev:
 593                     description: >-
 594                         A derived quantity. It is the standard deviation for the
 595                         values list, as computed by jumpavg library.
 596                     $ref: "#/$defs/types/nonnegative_number"
 597                 unit:
 598                     description: >-
 599                         Unit of measurement for the values.
 600                         Context and allowed values are specified elsewhere.
 601                     type: string
 602             required:
 603             -   values
 604             -   avg
 605             -   stdev
 606             -   unit
 607         rate_list_without_bandwidth:
 608             description: >-
 609                 Reusable composite type, multiple rate values.
 610             allOf:
 611             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 612             -   properties:
 613                     values:
 614                         items:
 615                             description: >-
 616                                 Unless specified otherwise,
 617                                 this is the aggregated rate
 618                                 (sum of both traffic directions).
 619                                 Depending on the usage, the value can express
 620                                 intended load, offered load, receive rate,
 621                                 and various approximations or estimated bounds
 622                                 thereof.
 623                     unit:
 624                         $ref: "#/$defs/types/rate_unit"
 625         bandwidth_list:
 626             description: >-
 627                 Reusable composite type, multiple bandwidth values. This is a
 628                 derived quantity.
 629             allOf:
 630             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 631             -   properties:
 632                     values:
 633                         items:
 634                             description: >-
 635                                 Unless specified otherwise,
 636                                 this is the aggregated bandwidth
 637                                 (sum of both traffic directions).
 638                                 Depending on the usage, the value can express
 639                                 intended load, offered load, receive rate,
 640                                 and various approximations or estimated bounds
 641                                 thereof.
 642                     unit:
 643                         $ref: "#/$defs/types/bandwidth_unit"
 644         rate_list_with_bandwidth:
 645             description: >-
 646                 Reusable composite type, joining primary rates
 647                 with optional derived bandwidths (and stats).
 648                 No test types currently computes the bandwidth part.
 649             type: object
 650             additionalProperties: false
 651             properties:
 652                 rate:
 653                     $ref: "#/$defs/types/rate_list_without_bandwidth"
 654                 bandwidth:
 655                     $ref: "#/$defs/types/bandwidth_list"
 656             required:
 657             - rate
 658     macros:
 659         lower_and_upper_rate:
 660             type: object
 661             additionalProperties: false
 662             properties:
 663                 lower:
 664                     description: >-
 665                         The lower bound (or min_rate) for the estimate
 666                         of a particular searched value.
 667                     $ref: "#/$defs/types/rate_with_bandwidth"
 668                 upper:
 669                     description: >-
 670                         The upper bound (or max_rate) for the estimate
 671                         of a particular searched value.
 672                     $ref: "#/$defs/types/rate_with_bandwidth"
 673             required:
 674             -   lower
 675             -   upper
 676         latency_numbers:
 677             type: object
 678             additionalProperties: false
 679             properties:
 680                 min:
 681                     description: >-
 682                         Rounded minimal latency time measured in this trial.
 683                         See unit property for the unit of measurement.
 684                     $ref: "#/$defs/types/nonnegative_integer"
 685                 max:
 686                     description: >-
 687                         Rounded maximal latency time measured in this trial.
 688                         See unit property for the unit of measurement.
 689                         Zero value is not allowed, as that is one of symptoms
 690                         of Traffic Generator failing to get proper latency.
 691                     $ref: "#/$defs/types/positive_integer"
 692                 avg:
 693                     description: >-
 694                         Rounded average latency time measured in this trial.
 695                         See unit property for the unit of measurement.
 696                     $ref: "#/$defs/types/nonnegative_integer"
 697                 hdrh:
 698                     description: >-
 699                         Base64-encoded compressed representation of HDRHistogram
 700                         of all latency sample times encountered
 701                         in this latency trial.
 702                         See unit property for the unit of measurement.
 703                         Note that some bins can be several units wide.
 704                     type: string
 705                 unit:
 706                     description: >-
 707                         Unit of measurement for latency times.
 708                         Currently a constant, but later versions
 709                         of the model may allow more values.
 710                     type: string
 711                     enum:
 712                     -   us
 713             required:
 714             -   avg
 715             -   hdrh
 716             -   max
 717             -   min
 718             -   unit
 719         latency_for_loads:
 720             type: object
 721             additionalProperties: false
 722             properties:
 723                 pdr_0:
 724                     description: >-
 725                         Object related to latency measurement performed
 726                         at minimal rate (currently 9000 pps per direction).
 727                     $ref: "#/$defs/macros/latency_numbers"
 728                 pdr_10:
 729                     description: >-
 730                         Object related to latency measurement performed
 731                         at 10% of PDR lower bound, if needed rounded up
 732                         to minimal rate (currently 9000 pps per direction).
 733                     $ref: "#/$defs/macros/latency_numbers"
 734                 pdr_50:
 735                     description: >-
 736                         Object related to latency measurement performed
 737                         at 50% of PDR lower bound, if needed rounded up
 738                         to minimal rate (currently 9000 pps per direction).
 739                     $ref: "#/$defs/macros/latency_numbers"
 740                 pdr_90:
 741                     description: >-
 742                         Object related to latency measurement performed
 743                         at 90% of PDR lower bound, if needed rounded up
 744                         to minimal rate (currently 9000 pps per direction).
 745                     $ref: "#/$defs/macros/latency_numbers"
 746             required:
 747             -   pdr_0
 748             -   pdr_10
 749             -   pdr_50
 750             -   pdr_90